Cloud-on-k8s: Elasticsearch init container conflict with Istio CNI init container

Created on 13 Jul 2020 · 2Comments · Source: elastic/cloud-on-k8s

When Istio CNI is enabled, Elasticsearch fails to start with the following error:

/usr/local/bin/docker-entrypoint.sh: line 49: /usr/share/elasticsearch/bin/elasticsearch-env-from-file: No such file or directory

This appears to be a conflict between the Istio CNI validation init container and the Elasticsearch file system prep init container. Requires further investigation to ascertain the root cause.

>bug

Source

charith-elastic

Most helpful comment

I have tracked down the problem to the pod template validation logic used in the reconciliation code for StatefulSets.

The pod template validation function is run before any CREATE or UPDATE operation is applied to a STS. It takes the pod template of the expected STS and performs a dry run of the CREATE operation.

When Istio CNI is enabled, the mutating webhook injects an init container into the pod to perform a validation check. The patch to do this inserts the istio-validation container as the first init container (/spec/initContainers/0).

Full patch from mutating webhook

[
  {
    "op": "add",
    "path": "/spec/initContainers/0",
    "value": {
      "name": "istio-validation",
      "image": "docker.io/istio/proxyv2:1.6.5",
      "args": [
        "istio-iptables",
        "-p",
        "15001",
        "-z",
        "15006",
        "-u",
        "1337",
        "-m",
        "REDIRECT",
        "-i",
        "*",
        "-x",
        "",
        "-b",
        "*",
        "-d",
        "15090,15021,9300,15020",
        "-o",
        "9300",
        "--run-validation",
        "--skip-rule-apply"
      ],
      "env": [
        {
          "name": "DNS_AGENT"
        }
      ],
      "resources": {
        "limits": {
          "cpu": "100m",
          "memory": "50Mi"
        },
        "requests": {
          "cpu": "10m",
          "memory": "10Mi"
        }
      },
      "imagePullPolicy": "Always",
      "securityContext": {
        "capabilities": {
          "drop": [
            "ALL"
          ]
        },
        "privileged": false,
        "runAsUser": 1337,
        "runAsGroup": 1337,
        "runAsNonRoot": true,
        "readOnlyRootFilesystem": true,
        "allowPrivilegeEscalation": false
      }
    }
  },
  {
    "op": "add",
    "path": "/spec/containers/-",
    "value": {
      "name": "istio-proxy",
      "image": "docker.io/istio/proxyv2:1.6.5",
      "args": [
        "proxy",
        "sidecar",
        "--domain",
        "$(POD_NAMESPACE).svc.cluster.local",
        "--serviceCluster",
        "elastic-istio.$(POD_NAMESPACE)",
        "--proxyLogLevel=warning",
        "--proxyComponentLogLevel=misc:error",
        "--trust-domain=cluster.local",
        "--concurrency",
        "2"
      ],
      "ports": [
        {
          "name": "http-envoy-prom",
          "containerPort": 15090,
          "protocol": "TCP"
        }
      ],
      "env": [
        {
          "name": "JWT_POLICY",
          "value": "third-party-jwt"
        },
        {
          "name": "PILOT_CERT_PROVIDER",
          "value": "istiod"
        },
        {
          "name": "CA_ADDR",
          "value": "istiod.istio-system.svc:15012"
        },
        {
          "name": "POD_NAME",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "metadata.name"
            }
          }
        },
        {
          "name": "POD_NAMESPACE",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "metadata.namespace"
            }
          }
        },
        {
          "name": "INSTANCE_IP",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "status.podIP"
            }
          }
        },
        {
          "name": "SERVICE_ACCOUNT",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "spec.serviceAccountName"
            }
          }
        },
        {
          "name": "HOST_IP",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "status.hostIP"
            }
          }
        },
        {
          "name": "CANONICAL_SERVICE",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "metadata.labels['service.istio.io/canonical-name']"
            }
          }
        },
        {
          "name": "CANONICAL_REVISION",
          "valueFrom": {
            "fieldRef": {
              "fieldPath": "metadata.labels['service.istio.io/canonical-revision']"
            }
          }
        },
        {
          "name": "PROXY_CONFIG",
          "value": "{\"proxyMetadata\":{\"DNS_AGENT\":\"\"}}\n"
        },
        {
          "name": "ISTIO_META_POD_PORTS",
          "value": "[\n    {\"name\":\"http\",\"containerPort\":9200,\"protocol\":\"TCP\"}\n    ,{\"name\":\"transport\",\"containerPort\":9300,\"protocol\":\"TCP\"}\n]"
        },
        {
          "name": "ISTIO_META_APP_CONTAINERS",
          "value": "[\n    elasticsearch\n]"
        },
        {
          "name": "ISTIO_META_CLUSTER_ID",
          "value": "Kubernetes"
        },
        {
          "name": "ISTIO_META_INTERCEPTION_MODE",
          "value": "REDIRECT"
        },
        {
          "name": "ISTIO_METAJSON_ANNOTATIONS",
          "value": "{\"co.elastic.logs/module\":\"elasticsearch\",\"traffic.sidecar.istio.io/excludeInboundPorts\":\"9300\",\"traffic.sidecar.istio.io/excludeOutboundPorts\":\"9300\",\"traffic.sidecar.istio.io/includeInboundPorts\":\"*\"}\n"
        },
        {
          "name": "ISTIO_META_WORKLOAD_NAME",
          "value": "elastic-istio-es-default-dummy-hknz7"
        },
        {
          "name": "ISTIO_META_OWNER",
          "value": "kubernetes://apis/v1/namespaces/istio-apps/pods/elastic-istio-es-default-dummy-hknz7"
        },
        {
          "name": "ISTIO_META_MESH_ID",
          "value": "cluster.local"
        },
        {
          "name": "DNS_AGENT"
        },
        {
          "name": "ISTIO_KUBE_APP_PROBERS",
          "value": "{}"
        }
      ],
      "resources": {
        "limits": {
          "cpu": "2",
          "memory": "1Gi"
        },
        "requests": {
          "cpu": "100m",
          "memory": "128Mi"
        }
      },
      "volumeMounts": [
        {
          "name": "istiod-ca-cert",
          "mountPath": "/var/run/secrets/istio"
        },
        {
          "name": "istio-data",
          "mountPath": "/var/lib/istio/data"
        },
        {
          "name": "istio-envoy",
          "mountPath": "/etc/istio/proxy"
        },
        {
          "name": "istio-token",
          "mountPath": "/var/run/secrets/tokens"
        },
        {
          "name": "istio-podinfo",
          "mountPath": "/etc/istio/pod"
        }
      ],
      "readinessProbe": {
        "httpGet": {
          "path": "/healthz/ready",
          "port": 15021
        },
        "initialDelaySeconds": 1,
        "periodSeconds": 2,
        "failureThreshold": 30
      },
      "imagePullPolicy": "Always",
      "securityContext": {
        "capabilities": {
          "drop": [
            "ALL"
          ]
        },
        "privileged": false,
        "runAsUser": 1337,
        "runAsGroup": 1337,
        "runAsNonRoot": true,
        "readOnlyRootFilesystem": true,
        "allowPrivilegeEscalation": false
      }
    }
  },
  {
    "op": "add",
    "path": "/spec/volumes/-",
    "value": {
      "name": "istio-envoy",
      "emptyDir": {
        "medium": "Memory"
      }
    }
  },
  {
    "op": "add",
    "path": "/spec/volumes/-",
    "value": {
      "name": "istio-data",
      "emptyDir": {}
    }
  },
  {
    "op": "add",
    "path": "/spec/volumes/-",
    "value": {
      "name": "istio-podinfo",
      "downwardAPI": {
        "items": [
          {
            "path": "labels",
            "fieldRef": {
              "fieldPath": "metadata.labels"
            }
          },
          {
            "path": "annotations",
            "fieldRef": {
              "fieldPath": "metadata.annotations"
            }
          }
        ]
      }
    }
  },
  {
    "op": "add",
    "path": "/spec/volumes/-",
    "value": {
      "name": "istio-token",
      "projected": {
        "sources": [
          {
            "serviceAccountToken": {
              "audience": "istio-ca",
              "expirationSeconds": 43200,
              "path": "istio-token"
            }
          }
        ]
      }
    }
  },
  {
    "op": "add",
    "path": "/spec/volumes/-",
    "value": {
      "name": "istiod-ca-cert",
      "configMap": {
        "name": "istio-ca-root-cert"
      }
    }
  },
  {
    "op": "add",
    "path": "/spec/securityContext",
    "value": {
      "fsGroup": 1337
    }
  },
  {
    "op": "add",
    "path": "/metadata/annotations/sidecar.istio.io~1status",
    "value": "{\"version\":\"ed4e6e8ed4ffa03fe7d5b9d2c27cc8c478625ade64be2859cae3da0db9e5ee2e\",\"initContainers\":[\"istio-validation\"],\"containers\":[\"istio-proxy\"],\"volumes\":[\"istio-envoy\",\"istio-data\",\"istio-podinfo\",\"istio-token\",\"istiod-ca-cert\"],\"imagePullSecrets\":null}"
  },
  {
    "op": "add",
    "path": "/metadata/labels/istio.io~1rev",
    "value": "default"
  },
  {
    "op": "add",
    "path": "/metadata/labels/security.istio.io~1tlsMode",
    "value": "istio"
  },
  {
    "op": "add",
    "path": "/metadata/labels/service.istio.io~1canonical-name",
    "value": "elastic-istio"
  },
  {
    "op": "add",
    "path": "/metadata/labels/service.istio.io~1canonical-revision",
    "value": "7.8.0"
  }
]

The Kube client modifies the request object in place.

As the validation code performs the dry run request using the template from the expected STS, the first entry in the initContainers array gets clobbered by that update (verified through a debugger). Even though the STS object is passed by value, arrays contained within it are still pointers -- so this makes sense.

Then, in the reconciliation code, we copy the expected object via reflection to the reconciled object and submit it as a CREATE request. This request now contains the clobbered init containers array, so the resulting pod is unable to properly prepare the file system.

Clobbered pod definition

{
  "apiVersion": "v1",
  "kind": "Pod",
  "metadata": {
    "annotations": {
      "cni.projectcalico.org/podIP": "10.24.14.95/32",
      "co.elastic.logs/module": "elasticsearch",
      "sidecar.istio.io/status": "{\"version\":\"ed4e6e8ed4ffa03fe7d5b9d2c27cc8c478625ade64be2859cae3da0db9e5ee2e\",\"initContainers\":[\"istio-validation\"],\"containers\":[\"istio-proxy\"],\"volumes\":[\"istio-envoy\",\"istio-data\",\"istio-podinfo\",\"istio-token\",\"istiod-ca-cert\"],\"imagePullSecrets\":null}",
      "traffic.sidecar.istio.io/excludeInboundPorts": "9300",
      "traffic.sidecar.istio.io/excludeOutboundPorts": "9300",
      "traffic.sidecar.istio.io/includeInboundPorts": "*",
      "update.k8s.elastic.co/timestamp": "2020-07-23T14:56:53.088719036Z"
    },
    "creationTimestamp": "2020-07-23T14:56:32Z",
    "generateName": "elastic-istio-es-default-",
    "labels": {
      "app": "elastic-istio",
      "common.k8s.elastic.co/type": "elasticsearch",
      "controller-revision-hash": "elastic-istio-es-default-7457c79f9d",
      "elasticsearch.k8s.elastic.co/cluster-name": "elastic-istio",
      "elasticsearch.k8s.elastic.co/config-hash": "3900325270",
      "elasticsearch.k8s.elastic.co/http-scheme": "http",
      "elasticsearch.k8s.elastic.co/node-data": "true",
      "elasticsearch.k8s.elastic.co/node-ingest": "true",
      "elasticsearch.k8s.elastic.co/node-master": "true",
      "elasticsearch.k8s.elastic.co/node-ml": "true",
      "elasticsearch.k8s.elastic.co/statefulset-name": "elastic-istio-es-default",
      "elasticsearch.k8s.elastic.co/version": "7.8.0",
      "istio.io/rev": "default",
      "security.istio.io/tlsMode": "istio",
      "service.istio.io/canonical-name": "elastic-istio",
      "service.istio.io/canonical-revision": "7.8.0",
      "statefulset.kubernetes.io/pod-name": "elastic-istio-es-default-0",
      "version": "7.8.0"
    },
    "name": "elastic-istio-es-default-0",
    "namespace": "istio-apps",
    "ownerReferences": [
      {
        "apiVersion": "apps/v1",
        "blockOwnerDeletion": true,
        "controller": true,
        "kind": "StatefulSet",
        "name": "elastic-istio-es-default",
        "uid": "42856fa5-d5d7-4872-b874-e311d44c497f"
      }
    ],
    "resourceVersion": "138050238",
    "selfLink": "/api/v1/namespaces/istio-apps/pods/elastic-istio-es-default-0",
    "uid": "7bf065a6-94f5-49d4-abc0-4866e73c24f5"
  },
  "spec": {
    "affinity": {
      "podAntiAffinity": {
        "preferredDuringSchedulingIgnoredDuringExecution": [
          {
            "podAffinityTerm": {
              "labelSelector": {
                "matchLabels": {
                  "elasticsearch.k8s.elastic.co/cluster-name": "elastic-istio"
                }
              },
              "topologyKey": "kubernetes.io/hostname"
            },
            "weight": 100
          }
        ]
      }
    },
    "automountServiceAccountToken": false,
    "containers": [
      {
        "env": [
          {
            "name": "POD_IP",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "status.podIP"
              }
            }
          },
          {
            "name": "POD_NAME",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.name"
              }
            }
          },
          {
            "name": "PROBE_PASSWORD_PATH",
            "value": "/mnt/elastic-internal/probe-user/elastic-internal-probe"
          },
          {
            "name": "PROBE_USERNAME",
            "value": "elastic-internal-probe"
          },
          {
            "name": "READINESS_PROBE_PROTOCOL",
            "value": "http"
          },
          {
            "name": "HEADLESS_SERVICE_NAME",
            "value": "elastic-istio-es-default"
          },
          {
            "name": "NSS_SDB_USE_CACHE",
            "value": "no"
          }
        ],
        "image": "docker.elastic.co/elasticsearch/elasticsearch:7.8.0",
        "imagePullPolicy": "IfNotPresent",
        "lifecycle": {
          "preStop": {
            "exec": {
              "command": [
                "bash",
                "-c",
                "/mnt/elastic-internal/scripts/pre-stop-hook-script.sh"
              ]
            }
          }
        },
        "name": "elasticsearch",
        "ports": [
          {
            "containerPort": 9200,
            "name": "http",
            "protocol": "TCP"
          },
          {
            "containerPort": 9300,
            "name": "transport",
            "protocol": "TCP"
          }
        ],
        "readinessProbe": {
          "exec": {
            "command": [
              "bash",
              "-c",
              "/mnt/elastic-internal/scripts/readiness-probe-script.sh"
            ]
          },
          "failureThreshold": 3,
          "initialDelaySeconds": 10,
          "periodSeconds": 5,
          "successThreshold": 1,
          "timeoutSeconds": 5
        },
        "resources": {
          "limits": {
            "memory": "2Gi"
          },
          "requests": {
            "memory": "2Gi"
          }
        },
        "terminationMessagePath": "/dev/termination-log",
        "terminationMessagePolicy": "File",
        "volumeMounts": [
          {
            "mountPath": "/mnt/elastic-internal/downward-api",
            "name": "downward-api",
            "readOnly": true
          },
          {
            "mountPath": "/usr/share/elasticsearch/bin",
            "name": "elastic-internal-elasticsearch-bin-local"
          },
          {
            "mountPath": "/mnt/elastic-internal/elasticsearch-config",
            "name": "elastic-internal-elasticsearch-config",
            "readOnly": true
          },
          {
            "mountPath": "/usr/share/elasticsearch/config",
            "name": "elastic-internal-elasticsearch-config-local"
          },
          {
            "mountPath": "/usr/share/elasticsearch/plugins",
            "name": "elastic-internal-elasticsearch-plugins-local"
          },
          {
            "mountPath": "/usr/share/elasticsearch/config/http-certs",
            "name": "elastic-internal-http-certificates",
            "readOnly": true
          },
          {
            "mountPath": "/mnt/elastic-internal/probe-user",
            "name": "elastic-internal-probe-user",
            "readOnly": true
          },
          {
            "mountPath": "/usr/share/elasticsearch/config/transport-remote-certs/",
            "name": "elastic-internal-remote-certificate-authorities",
            "readOnly": true
          },
          {
            "mountPath": "/mnt/elastic-internal/scripts",
            "name": "elastic-internal-scripts",
            "readOnly": true
          },
          {
            "mountPath": "/usr/share/elasticsearch/config/transport-certs",
            "name": "elastic-internal-transport-certificates",
            "readOnly": true
          },
          {
            "mountPath": "/mnt/elastic-internal/unicast-hosts",
            "name": "elastic-internal-unicast-hosts",
            "readOnly": true
          },
          {
            "mountPath": "/mnt/elastic-internal/xpack-file-realm",
            "name": "elastic-internal-xpack-file-realm",
            "readOnly": true
          },
          {
            "mountPath": "/usr/share/elasticsearch/data",
            "name": "elasticsearch-data"
          },
          {
            "mountPath": "/usr/share/elasticsearch/logs",
            "name": "elasticsearch-logs"
          }
        ]
      },
      {
        "args": [
          "proxy",
          "sidecar",
          "--domain",
          "$(POD_NAMESPACE).svc.cluster.local",
          "--serviceCluster",
          "elastic-istio.$(POD_NAMESPACE)",
          "--proxyLogLevel=warning",
          "--proxyComponentLogLevel=misc:error",
          "--trust-domain=cluster.local",
          "--concurrency",
          "2"
        ],
        "env": [
          {
            "name": "JWT_POLICY",
            "value": "third-party-jwt"
          },
          {
            "name": "PILOT_CERT_PROVIDER",
            "value": "istiod"
          },
          {
            "name": "CA_ADDR",
            "value": "istiod.istio-system.svc:15012"
          },
          {
            "name": "POD_NAME",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.name"
              }
            }
          },
          {
            "name": "POD_NAMESPACE",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.namespace"
              }
            }
          },
          {
            "name": "INSTANCE_IP",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "status.podIP"
              }
            }
          },
          {
            "name": "SERVICE_ACCOUNT",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "spec.serviceAccountName"
              }
            }
          },
          {
            "name": "HOST_IP",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "status.hostIP"
              }
            }
          },
          {
            "name": "CANONICAL_SERVICE",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.labels['service.istio.io/canonical-name']"
              }
            }
          },
          {
            "name": "CANONICAL_REVISION",
            "valueFrom": {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.labels['service.istio.io/canonical-revision']"
              }
            }
          },
          {
            "name": "PROXY_CONFIG",
            "value": "{\"proxyMetadata\":{\"DNS_AGENT\":\"\"}}\n"
          },
          {
            "name": "ISTIO_META_POD_PORTS",
            "value": "[\n    {\"name\":\"http\",\"containerPort\":9200,\"protocol\":\"TCP\"}\n    ,{\"name\":\"transport\",\"containerPort\":9300,\"protocol\":\"TCP\"}\n]"
          },
          {
            "name": "ISTIO_META_APP_CONTAINERS",
            "value": "[\n    elasticsearch\n]"
          },
          {
            "name": "ISTIO_META_CLUSTER_ID",
            "value": "Kubernetes"
          },
          {
            "name": "ISTIO_META_INTERCEPTION_MODE",
            "value": "REDIRECT"
          },
          {
            "name": "ISTIO_METAJSON_ANNOTATIONS",
            "value": "{\"co.elastic.logs/module\":\"elasticsearch\",\"sidecar.istio.io/status\":\"{\\\"version\\\":\\\"ed4e6e8ed4ffa03fe7d5b9d2c27cc8c478625ade64be2859cae3da0db9e5ee2e\\\",\\\"initContainers\\\":[\\\"istio-validation\\\"],\\\"containers\\\":[\\\"istio-proxy\\\"],\\\"volumes\\\":[\\\"istio-envoy\\\",\\\"istio-data\\\",\\\"istio-podinfo\\\",\\\"istio-token\\\",\\\"istiod-ca-cert\\\"],\\\"imagePullSecrets\\\":null}\",\"traffic.sidecar.istio.io/excludeInboundPorts\":\"9300\",\"traffic.sidecar.istio.io/excludeOutboundPorts\":\"9300\",\"traffic.sidecar.istio.io/includeInboundPorts\":\"*\"}\n"
          },
          {
            "name": "ISTIO_META_WORKLOAD_NAME",
            "value": "elastic-istio-es-default"
          },
          {
            "name": "ISTIO_META_OWNER",
            "value": "kubernetes://apis/apps/v1/namespaces/istio-apps/statefulsets/elastic-istio-es-default"
          },
          {
            "name": "ISTIO_META_MESH_ID",
            "value": "cluster.local"
          },
          {
            "name": "DNS_AGENT"
          },
          {
            "name": "ISTIO_KUBE_APP_PROBERS",
            "value": "{}"
          }
        ],
        "image": "docker.io/istio/proxyv2:1.6.5",
        "imagePullPolicy": "Always",
        "name": "istio-proxy",
        "ports": [
          {
            "containerPort": 15090,
            "name": "http-envoy-prom",
            "protocol": "TCP"
          }
        ],
        "readinessProbe": {
          "failureThreshold": 30,
          "httpGet": {
            "path": "/healthz/ready",
            "port": 15021,
            "scheme": "HTTP"
          },
          "initialDelaySeconds": 1,
          "periodSeconds": 2,
          "successThreshold": 1,
          "timeoutSeconds": 1
        },
        "resources": {
          "limits": {
            "cpu": "2",
            "memory": "1Gi"
          },
          "requests": {
            "cpu": "100m",
            "memory": "128Mi"
          }
        },
        "securityContext": {
          "allowPrivilegeEscalation": false,
          "capabilities": {
            "drop": [
              "ALL"
            ]
          },
          "privileged": false,
          "readOnlyRootFilesystem": true,
          "runAsGroup": 1337,
          "runAsNonRoot": true,
          "runAsUser": 1337
        },
        "terminationMessagePath": "/dev/termination-log",
        "terminationMessagePolicy": "File",
        "volumeMounts": [
          {
            "mountPath": "/var/run/secrets/istio",
            "name": "istiod-ca-cert"
          },
          {
            "mountPath": "/var/lib/istio/data",
            "name": "istio-data"
          },
          {
            "mountPath": "/etc/istio/proxy",
            "name": "istio-envoy"
          },
          {
            "mountPath": "/var/run/secrets/tokens",
            "name": "istio-token"
          },
          {
            "mountPath": "/etc/istio/pod",
            "name": "istio-podinfo"
          }
        ]
      }
    ],
    "dnsPolicy": "ClusterFirst",
    "enableServiceLinks": true,
    "hostname": "elastic-istio-es-default-0",
    "initContainers": [
      {
        "args": [
          "istio-iptables",
          "-p",
          "15001",
          "-z",
          "15006",
          "-u",
          "1337",
          "-m",
          "REDIRECT",
          "-i",
          "*",
          "-x",
          "",
          "-b",
          "*",
          "-d",
          "15090,15021,9300,15020",
          "-o",
          "9300",
          "--run-validation",
          "--skip-rule-apply"
        ],
        "env": [
          {
            "name": "DNS_AGENT"
          }
        ],
        "image": "docker.io/istio/proxyv2:1.6.5",
        "imagePullPolicy": "Always",
        "name": "istio-validation",
        "resources": {
          "limits": {
            "cpu": "100m",
            "memory": "50Mi"
          },
          "requests": {
            "cpu": "10m",
            "memory": "10Mi"
          }
        },
        "securityContext": {
          "allowPrivilegeEscalation": false,
          "capabilities": {
            "drop": [
              "ALL"
            ]
          },
          "privileged": false,
          "readOnlyRootFilesystem": true,
          "runAsGroup": 1337,
          "runAsNonRoot": true,
          "runAsUser": 1337
        },
        "terminationMessagePath": "/dev/termination-log",
        "terminationMessagePolicy": "File"
      }
    ],
    "nodeName": "gke-cell-scale-test-clust-dev-control-a1fa4a65-n9pk",
    "priority": 0,
    "restartPolicy": "Always",
    "schedulerName": "default-scheduler",
    "securityContext": {
      "fsGroup": 1337
    },
    "serviceAccount": "default",
    "serviceAccountName": "default",
    "subdomain": "elastic-istio-es-default",
    "terminationGracePeriodSeconds": 180,
    "tolerations": [
      {
        "effect": "NoExecute",
        "key": "node.kubernetes.io/not-ready",
        "operator": "Exists",
        "tolerationSeconds": 300
      },
      {
        "effect": "NoExecute",
        "key": "node.kubernetes.io/unreachable",
        "operator": "Exists",
        "tolerationSeconds": 300
      }
    ],
    "volumes": [
      {
        "name": "elasticsearch-data",
        "persistentVolumeClaim": {
          "claimName": "elasticsearch-data-elastic-istio-es-default-0"
        }
      },
      {
        "downwardAPI": {
          "defaultMode": 420,
          "items": [
            {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.labels"
              },
              "path": "labels"
            }
          ]
        },
        "name": "downward-api"
      },
      {
        "emptyDir": {},
        "name": "elastic-internal-elasticsearch-bin-local"
      },
      {
        "name": "elastic-internal-elasticsearch-config",
        "secret": {
          "defaultMode": 420,
          "optional": false,
          "secretName": "elastic-istio-es-default-es-config"
        }
      },
      {
        "emptyDir": {},
        "name": "elastic-internal-elasticsearch-config-local"
      },
      {
        "emptyDir": {},
        "name": "elastic-internal-elasticsearch-plugins-local"
      },
      {
        "name": "elastic-internal-http-certificates",
        "secret": {
          "defaultMode": 420,
          "optional": false,
          "secretName": "elastic-istio-es-http-certs-internal"
        }
      },
      {
        "name": "elastic-internal-probe-user",
        "secret": {
          "defaultMode": 420,
          "items": [
            {
              "key": "elastic-internal-probe",
              "path": "elastic-internal-probe"
            }
          ],
          "optional": false,
          "secretName": "elastic-istio-es-internal-users"
        }
      },
      {
        "name": "elastic-internal-remote-certificate-authorities",
        "secret": {
          "defaultMode": 420,
          "optional": false,
          "secretName": "elastic-istio-es-remote-ca"
        }
      },
      {
        "configMap": {
          "defaultMode": 493,
          "name": "elastic-istio-es-scripts",
          "optional": false
        },
        "name": "elastic-internal-scripts"
      },
      {
        "name": "elastic-internal-transport-certificates",
        "secret": {
          "defaultMode": 420,
          "optional": false,
          "secretName": "elastic-istio-es-transport-certificates"
        }
      },
      {
        "configMap": {
          "defaultMode": 420,
          "name": "elastic-istio-es-unicast-hosts",
          "optional": false
        },
        "name": "elastic-internal-unicast-hosts"
      },
      {
        "name": "elastic-internal-xpack-file-realm",
        "secret": {
          "defaultMode": 420,
          "optional": false,
          "secretName": "elastic-istio-es-xpack-file-realm"
        }
      },
      {
        "emptyDir": {},
        "name": "elasticsearch-logs"
      },
      {
        "emptyDir": {
          "medium": "Memory"
        },
        "name": "istio-envoy"
      },
      {
        "emptyDir": {},
        "name": "istio-data"
      },
      {
        "downwardAPI": {
          "defaultMode": 420,
          "items": [
            {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.labels"
              },
              "path": "labels"
            },
            {
              "fieldRef": {
                "apiVersion": "v1",
                "fieldPath": "metadata.annotations"
              },
              "path": "annotations"
            }
          ]
        },
        "name": "istio-podinfo"
      },
      {
        "name": "istio-token",
        "projected": {
          "defaultMode": 420,
          "sources": [
            {
              "serviceAccountToken": {
                "audience": "istio-ca",
                "expirationSeconds": 43200,
                "path": "istio-token"
              }
            }
          ]
        }
      },
      {
        "configMap": {
          "defaultMode": 420,
          "name": "istio-ca-root-cert"
        },
        "name": "istiod-ca-cert"
      }
    ]
  },
  "status": {
    "conditions": [
      {
        "lastProbeTime": null,
        "lastTransitionTime": "2020-07-23T14:56:44Z",
        "status": "True",
        "type": "Initialized"
      },
      {
        "lastProbeTime": null,
        "lastTransitionTime": "2020-07-23T14:56:32Z",
        "message": "containers with unready status: [elasticsearch]",
        "reason": "ContainersNotReady",
        "status": "False",
        "type": "Ready"
      },
      {
        "lastProbeTime": null,
        "lastTransitionTime": "2020-07-23T14:56:32Z",
        "message": "containers with unready status: [elasticsearch]",
        "reason": "ContainersNotReady",
        "status": "False",
        "type": "ContainersReady"
      },
      {
        "lastProbeTime": null,
        "lastTransitionTime": "2020-07-23T14:56:32Z",
        "status": "True",
        "type": "PodScheduled"
      }
    ],
    "containerStatuses": [
      {
        "containerID": "docker://7c5fe803ed2ed522922c316fb9f02241b38158ddc2f4d2c77430a2a0f1e65788",
        "image": "docker.elastic.co/elasticsearch/elasticsearch:7.8.0",
        "imageID": "docker-pullable://docker.elastic.co/elasticsearch/elasticsearch@sha256:161bc8c7054c622b057324618a2e8bc49ae703e64901b141a16d9c8bdd3b82f9",
        "lastState": {
          "terminated": {
            "containerID": "docker://7c5fe803ed2ed522922c316fb9f02241b38158ddc2f4d2c77430a2a0f1e65788",
            "exitCode": 1,
            "finishedAt": "2020-07-23T14:56:46Z",
            "reason": "Error",
            "startedAt": "2020-07-23T14:56:46Z"
          }
        },
        "name": "elasticsearch",
        "ready": false,
        "restartCount": 1,
        "started": false,
        "state": {
          "waiting": {
            "message": "back-off 10s restarting failed container=elasticsearch pod=elastic-istio-es-default-0_istio-apps(7bf065a6-94f5-49d4-abc0-4866e73c24f5)",
            "reason": "CrashLoopBackOff"
          }
        }
      },
      {
        "containerID": "docker://4f78772edc089acd7e2d4b7587419ec8866100bbb818d39bca2081a2121b4683",
        "image": "istio/proxyv2:1.6.5",
        "imageID": "docker-pullable://istio/proxyv2@sha256:ec2df06d76e8845fbce0ac1b4b85ab06a7beabab8a69fcc3bb2b573378b71c47",
        "lastState": {},
        "name": "istio-proxy",
        "ready": true,
        "restartCount": 0,
        "started": true,
        "state": {
          "running": {
            "startedAt": "2020-07-23T14:56:45Z"
          }
        }
      }
    ],
    "hostIP": "10.132.0.7",
    "initContainerStatuses": [
      {
        "containerID": "docker://e8d4cd8a9199c7770f97b9ffd444056578813cdee8f02642afbf1d9bd88bf3f9",
        "image": "istio/proxyv2:1.6.5",
        "imageID": "docker-pullable://istio/proxyv2@sha256:ec2df06d76e8845fbce0ac1b4b85ab06a7beabab8a69fcc3bb2b573378b71c47",
        "lastState": {},
        "name": "istio-validation",
        "ready": true,
        "restartCount": 0,
        "state": {
          "terminated": {
            "containerID": "docker://e8d4cd8a9199c7770f97b9ffd444056578813cdee8f02642afbf1d9bd88bf3f9",
            "exitCode": 0,
            "finishedAt": "2020-07-23T14:56:43Z",
            "reason": "Completed",
            "startedAt": "2020-07-23T14:56:43Z"
          }
        }
      }
    ],
    "phase": "Running",
    "podIP": "10.24.14.95",
    "podIPs": [
      {
        "ip": "10.24.14.95"
      }
    ],
    "qosClass": "Burstable",
    "startTime": "2020-07-23T14:56:32Z"
  }
}

charith-elastic on 23 Jul 2020

❤4 👍1

All 2 comments

Creating the Elasticsearch StatefulSet manually using kubectl works as expected (both istio-validation and elastic-internal-init-filesystem containers are in the init containers array). When the operator creates the StatefulSet, elastic-internal-init-filesystem container is somehow lost and only the istio-validation container is left in the init container list. This suggests some weird interaction between the Istio webhook and the lifecycle of objects created by the controller-runtime client.

charith-elastic on 15 Jul 2020