Falco: Failure to find a BPF probe on Container Optimized OS

Created on 6 Jun 2019 · 20Comments · Source: falcosecurity/falco

I'm having the issues running Falco on GCP using Kubernetes Engine with Container Optimized OS.

Initially, I assumed the error was related to not having the host /etc mounted on the container or not having the SYSDIG_BPF_PROBE environment variable exported, which I was able to confirm that both are correct on the container.

To spin up this infrastructure I've converted the yaml files from the examples to terraform:

resource "kubernetes_service_account" "falco_sa" {
  metadata {
    name = "falco-account"
    labels = {
      app  = "falco"
      role = "security"
    }
  }
}

resource "kubernetes_cluster_role" "falco_cr" {
  metadata {
    name = "falco-cluster-role"
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  rule {
    api_groups = ["extensions", ""]
    resources  = ["nodes", "namespaces", "pods", "replicationcontrollers", "replicasets", "services", "daemonsets", "deployments", "events", "configmaps"]
    verbs      = ["get", "list", "watch"]
  }
  rule {
    non_resource_urls = ["/healthz", "/healthz/*"]
    verbs             = ["get"]
  }
}

resource "kubernetes_cluster_role_binding" "falco_crb" {
  metadata {
    name = "falco-cluster-role-bind"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  subject {
    kind      = "ServiceAccount"
    name      = kubernetes_service_account.falco_sa.metadata.0.name
    namespace = "default"
  }

  role_ref {
    kind      = "ClusterRole"
    name      = kubernetes_cluster_role.falco_cr.metadata.0.name
    api_group = "rbac.authorization.k8s.io"
  }
}

resource "kubernetes_config_map" "falco_cfgmap" {
  metadata {
    name = "falco-cfgmap"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  data = {
    "application_rules.yaml" = file("configs/falco/application_rules.yaml")
    "falco_rules.local.yaml" = file("configs/falco/falco_rules.local.yaml")
    "falco_rules.yaml"       = file("configs/falco/falco_rules.yaml")
    "k8s_audit_rules.yaml"   = file("configs/falco/k8s_audit_rules.yaml")
  }
}

resource "kubernetes_daemonset" "falco_ds" {
  metadata {
    name = "falco-daemonset"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  spec {

    selector {
      match_labels = {
        app  = "falco"
        role = "security"
      }
    }


    template {
      metadata {
        labels = {
          app  = "falco"
          role = "security"
        }
      }

      spec {
        service_account_name = kubernetes_service_account.falco_sa.metadata.0.name
        volume {
          name = "docker-socket"
          host_path {
            path = "/var/run/docker.socket"
          }
        }
        volume {
          name = "containerd-socket"
          host_path {
            path = "/run/containerd/containerd.sock"
          }
        }
        volume {
          name = "dev-fs"
          host_path {
            path = "/dev"
          }
        }
        volume {
          name = "proc-fs"
          host_path {
            path = "/proc"
          }
        }
        volume {
          name = "boot-fs"
          host_path {
            path = "/boot"
          }
        }
        volume {
          name = "lib-modules"
          host_path {
            path = "/lib/modules"
          }
        }
        volume {
          name = "usr-fs"
          host_path {
            path = "/usr"
          }
        }
        volume {
          name = "etc-fs"
          host_path {
            path = "/etc"
          }
        }
        volume {
          name = "falco-config"
          config_map {
            name = kubernetes_config_map.falco_cfgmap.metadata.0.name
          }
        }

        container {
          name  = "falco"
          image = "falcosecurity/falco:latest"
          args  = [
            "/usr/bin/falco",
            "--cri", "/host/run/containerd/containerd.sock",
            "-K", "/var/run/secrets/kubernetes.io/serviceaccount/token",
            "-k", "https://$(KUBERNETES_SERVICE_HOST)",
            "-pk",
          ]
          security_context {
            privileged = true
          }
          env {
            name  = "SYSDIG_BPF_PROBE"
            value = ""
          }
          volume_mount {
            name       = "docker-socket"
            mount_path = "/host/var/run/docker.sock"
          }
          volume_mount {
            name       = "containerd-socket"
            mount_path = "/host/run/containerd/containerd.sock"
          }
          volume_mount {
            name       = "dev-fs"
            mount_path = "/host/dev"
          }
          volume_mount {
            name       = "proc-fs"
            mount_path = "/host/proc"
            read_only = true
          }
          volume_mount {
            name       = "boot-fs"
            mount_path = "/host/boot"
            read_only = true
          }
          volume_mount {
            name       = "lib-modules"
            mount_path = "/host/lib/modules"
            read_only = true
          }
          volume_mount {
            name       = "usr-fs"
            mount_path = "/host/usr"
            read_only = true
          }
          volume_mount {
            name       = "etc-fs"
            mount_path = "/host/etc"
            read_only = true
          }
          volume_mount {
            name       = "falco-config"
            mount_path = "/etc/falco"
          }
        }
      }
    }
  }
}

resource "kubernetes_service" "falco_svc" {
  metadata {
    name = kubernetes_daemonset.falco_ds.metadata.0.name
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  spec {
    type = "ClusterIP"

    port {
      protocol = "TCP"
      port     = 8765
    }

    selector = {
      app  = "falco"
      role = "security"
    }
  }
}

The output of describe daemonset looks correct:

Name:           falco-daemonset
Selector:       app=falco,role=security
Node-Selector:  <none>
Labels:         app=falco
                role=security
Annotations:    deprecated.daemonset.template.generation: 4
Desired Number of Nodes Scheduled: 3
Current Number of Nodes Scheduled: 3
Number of Nodes Scheduled with Up-to-date Pods: 3
Number of Nodes Scheduled with Available Pods: 0
Number of Nodes Misscheduled: 0
Pods Status:  3 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
  Labels:           app=falco
                    role=security
  Service Account:  falco-account
  Containers:
   falco:
    Image:      falcosecurity/falco:latest
    Port:       <none>
    Host Port:  <none>
    Args:
      /usr/bin/falco
      --cri
      /host/run/containerd/containerd.sock
      -K
      /var/run/secrets/kubernetes.io/serviceaccount/token
      -k
      https://$(KUBERNETES_SERVICE_HOST)
      -pk
     Environment:
      SYSDIG_BPF_PROBE:  
    Mounts:
      /etc/falco from falco-config (rw)
      /host/boot from boot-fs (ro)
      /host/dev from dev-fs (rw)
      /host/etc from etc-fs (ro)
      /host/lib/modules from lib-modules (ro)
      /host/proc from proc-fs (ro)
      /host/run/containerd/containerd.sock from containerd-socket (rw)
      /host/usr from usr-fs (ro)
      /host/var/run/docker.sock from docker-socket (rw)
  Volumes:
   docker-socket:
    Type:          HostPath (bare host directory volume)
    Path:          /var/run/docker.socket
    HostPathType:  
   containerd-socket:
    Type:          HostPath (bare host directory volume)
    Path:          /run/containerd/containerd.sock
    HostPathType:  
   dev-fs:
    Type:          HostPath (bare host directory volume)
    Path:          /dev
    HostPathType:  
   proc-fs:
    Type:          HostPath (bare host directory volume)
    Path:          /proc
    HostPathType:  
   boot-fs:
    Type:          HostPath (bare host directory volume)
    Path:          /boot
    HostPathType:  
   lib-modules:
    Type:          HostPath (bare host directory volume)
    Path:          /lib/modules
    HostPathType:  
   usr-fs:
    Type:          HostPath (bare host directory volume)
    Path:          /usr
    HostPathType:  
   etc-fs:
    Type:          HostPath (bare host directory volume)
    Path:          /etc
    HostPathType:  
   falco-config:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      falco-cfgmap
    Optional:  false
Events:
  Type    Reason            Age    From                  Message
  ----    ------            ----   ----                  -------
  Normal  SuccessfulCreate  45m    daemonset-controller  Created pod: falco-daemonset-vvfm4
  Normal  SuccessfulCreate  45m    daemonset-controller  Created pod: falco-daemonset-r6xx2
  Normal  SuccessfulCreate  45m    daemonset-controller  Created pod: falco-daemonset-rk64p
  Normal  SuccessfulDelete  34m    daemonset-controller  Deleted pod: falco-daemonset-vvfm4
  Normal  SuccessfulDelete  34m    daemonset-controller  Deleted pod: falco-daemonset-r6xx2
  Normal  SuccessfulDelete  34m    daemonset-controller  Deleted pod: falco-daemonset-rk64p
  Normal  SuccessfulCreate  34m    daemonset-controller  Created pod: falco-daemonset-ghpbt
  Normal  SuccessfulCreate  34m    daemonset-controller  Created pod: falco-daemonset-6n7fr
  Normal  SuccessfulCreate  34m    daemonset-controller  Created pod: falco-daemonset-dz42d
  Normal  SuccessfulDelete  19m    daemonset-controller  Deleted pod: falco-daemonset-dz42d
  Normal  SuccessfulDelete  19m    daemonset-controller  Deleted pod: falco-daemonset-ghpbt
  Normal  SuccessfulDelete  19m    daemonset-controller  Deleted pod: falco-daemonset-6n7fr
  Normal  SuccessfulCreate  19m    daemonset-controller  Created pod: falco-daemonset-7l9wr
  Normal  SuccessfulCreate  19m    daemonset-controller  Created pod: falco-daemonset-4p9xg
  Normal  SuccessfulCreate  19m    daemonset-controller  Created pod: falco-daemonset-hn24r
  Normal  SuccessfulDelete  8m33s  daemonset-controller  Deleted pod: falco-daemonset-hn24r
  Normal  SuccessfulDelete  8m33s  daemonset-controller  Deleted pod: falco-daemonset-4p9xg
  Normal  SuccessfulDelete  8m33s  daemonset-controller  Deleted pod: falco-daemonset-7l9wr
  Normal  SuccessfulCreate  8m23s  daemonset-controller  Created pod: falco-daemonset-t7xgr
  Normal  SuccessfulCreate  8m23s  daemonset-controller  Created pod: falco-daemonset-ptpvj
  Normal  SuccessfulCreate  8m23s  daemonset-controller  Created pod: falco-daemonset-dkctn

And my logs:

* Setting up /usr/src links from host
* Mounting debugfs
Found kernel config at /proc/config.gz
* COS detected (build 11647.163.0), downloading and setting up kernel headers
* Downloading https://storage.googleapis.com/cos-tools/11647.163.0/kernel-src.tar.gz
* Extracting kernel sources
* Configuring kernel
scripts/sign-file.c:25:30: fatal error: openssl/opensslv.h: No such file or directory
compilation terminated.
make[1]: *** [scripts/Makefile.host:102: scripts/sign-file] Error 1
make: *** [Makefile:572: scripts] Error 2
* Trying to compile BPF probe falco-probe-bpf (falco-probe-bpf-0.15.0-x86_64-4.14.94+-d7aaf2e0f41bfe30d6d84fe8e754c0d9.o)
In file included from /usr/src/falco-0.15.0/bpf/probe.c:23:
/usr/src/falco-0.15.0/bpf/fillers.h:2017:26: error: no member named 'loginuid' in 'struct task_struct'
                loginuid = _READ(task->loginuid);
                                 ~~~~  ^
/usr/src/falco-0.15.0/bpf/plumbing_helpers.h:18:28: note: expanded from macro '_READ'
#define _READ(P) ({ typeof(P) _val;                             \
                           ^
In file included from /usr/src/falco-0.15.0/bpf/probe.c:23:
/usr/src/falco-0.15.0/bpf/fillers.h:2017:26: error: no member named 'loginuid' in 'struct task_struct'
                loginuid = _READ(task->loginuid);
                                 ~~~~  ^
/usr/src/falco-0.15.0/bpf/plumbing_helpers.h:20:44: note: expanded from macro '_READ'
                    bpf_probe_read(&_val, sizeof(_val), &P);    \
                                                         ^
In file included from /usr/src/falco-0.15.0/bpf/probe.c:23:
/usr/src/falco-0.15.0/bpf/fillers.h:2017:12: error: assigning to 'kuid_t' from incompatible type 'void'
                loginuid = _READ(task->loginuid);
                         ^ ~~~~~~~~~~~~~~~~~~~~~
3 errors generated.
make[2]: *** [/usr/src/falco-0.15.0/bpf/Makefile:33: /usr/src/falco-0.15.0/bpf/probe.o] Error 1
make[1]: *** [Makefile:1541: _module_/usr/src/falco-0.15.0/bpf] Error 2
make: *** [Makefile:18: all] Error 2
mv: cannot stat '/usr/src/falco-0.15.0/bpf/probe.o': No such file or directory
* Trying to download precompiled BPF probe from https://s3.amazonaws.com/download.draios.com/stable/sysdig-probe-binaries/falco-probe-bpf-0.15.0-x86_64-4.14.94%2B-d7aaf2e0f41bfe30d6d84fe8e754c0d9.o
curl: (22) The requested URL returned error: 404 Not Found
* Failure to find a BPF probe

arebpf-probe kinbug

Source

caquino

👍5

Most helpful comment

@mfdii I can confirm that with the KBUILD_EXTRA_CPPFLAGS the build works without issues.

It also required some changes to my terraform to work, to automount the service account secret.

I'm sharing the final configuration here in case anyone else is interested.

resource "kubernetes_service_account" "falco_sa" {
  metadata {
    name = "falco-account"
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  automount_service_account_token = true
}

resource "kubernetes_cluster_role" "falco_cr" {
  metadata {
    name = "falco-cluster-role"
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  rule {
    api_groups = ["extensions", ""]
    resources  = ["nodes", "namespaces", "pods", "replicationcontrollers", "replicasets", "services", "daemonsets", "deployments", "events", "configmaps"]
    verbs      = ["get", "list", "watch"]
  }
  rule {
    non_resource_urls = ["/healthz", "/healthz/*"]
    verbs             = ["get"]
  }
}

resource "kubernetes_cluster_role_binding" "falco_crb" {
  metadata {
    name = "falco-cluster-role-bind"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  subject {
    kind      = "ServiceAccount"
    name      = kubernetes_service_account.falco_sa.metadata.0.name
    namespace = "default"
  }

  role_ref {
    kind      = "ClusterRole"
    name      = kubernetes_cluster_role.falco_cr.metadata.0.name
    api_group = "rbac.authorization.k8s.io"
  }
}

resource "kubernetes_config_map" "falco_cfgmap" {
  metadata {
    name = "falco-cfgmap"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  data = {
    "application_rules.yaml" = file("configs/falco/application_rules.yaml")
    "falco_rules.local.yaml" = file("configs/falco/falco_rules.local.yaml")
    "falco_rules.yaml"       = file("configs/falco/falco_rules.yaml")
    "k8s_audit_rules.yaml"   = file("configs/falco/k8s_audit_rules.yaml")
    "falco.yaml"             = file("configs/falco/falco.yaml")
  }
}

resource "kubernetes_daemonset" "falco_ds" {
  metadata {
    name = "falco-daemonset"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  spec {

    selector {
      match_labels = {
        app  = "falco"
        role = "security"
      }
    }

    template {
      metadata {
        labels = {
          app  = "falco"
          role = "security"
        }
      }

      spec {
        host_network         = true
        service_account_name = kubernetes_service_account.falco_sa.metadata.0.name
        dns_policy           = "ClusterFirstWithHostNet"

        volume {
          name = "docker-socket"
          host_path {
            path = "/var/run/docker.socket"
          }
        }
        volume {
          name = "containerd-socket"
          host_path {
            path = "/run/containerd/containerd.sock"
          }
        }
        volume {
          name = "dev-fs"
          host_path {
            path = "/dev"
          }
        }
        volume {
          name = "proc-fs"
          host_path {
            path = "/proc"
          }
        }
        volume {
          name = "boot-fs"
          host_path {
            path = "/boot"
          }
        }
        volume {
          name = "lib-modules"
          host_path {
            path = "/lib/modules"
          }
        }
        volume {
          name = "usr-fs"
          host_path {
            path = "/usr"
          }
        }
        volume {
          name = "etc-fs"
          host_path {
            path = "/etc"
          }
        }
        volume {
          name = "dshm"
          empty_dir {
            medium = "Memory"
          }
        }
        volume {
          name = "falco-config"
          config_map {
            name = kubernetes_config_map.falco_cfgmap.metadata.0.name
          }
        }

        container {
          name  = "falco"
          image = "falcosecurity/falco:latest"
          args = [
            "/usr/bin/falco",
            "--cri", "/host/run/containerd/containerd.sock",
            "-K", "/var/run/secrets/kubernetes.io/serviceaccount/token",
            "-k", "https://$(KUBERNETES_SERVICE_HOST)",
            "-pk",
          ]
          security_context {
            privileged = true
          }
          env {
            name  = "SYSDIG_BPF_PROBE"
            value = ""
          }
          env {
            name  = "KBUILD_EXTRA_CPPFLAGS"
            value = "-DCOS_73_WORKAROUND"
          }
          volume_mount {
            name       = "docker-socket"
            mount_path = "/host/var/run/docker.sock"
          }
          volume_mount {
            name       = "containerd-socket"
            mount_path = "/host/run/containerd/containerd.sock"
          }
          volume_mount {
            name       = "dev-fs"
            mount_path = "/host/dev"
          }
          volume_mount {
            name       = "proc-fs"
            mount_path = "/host/proc"
            read_only  = true
          }
          volume_mount {
            name       = "boot-fs"
            mount_path = "/host/boot"
            read_only  = true
          }
          volume_mount {
            name       = "lib-modules"
            mount_path = "/host/lib/modules"
            read_only  = true
          }
          volume_mount {
            name       = "usr-fs"
            mount_path = "/host/usr"
            read_only  = true
          }
          volume_mount {
            name       = "etc-fs"
            mount_path = "/host/etc"
            read_only  = true
          }
          volume_mount {
            name       = "dshm"
            mount_path = "/dev/shm"
          }
          volume_mount {
            name       = "falco-config"
            mount_path = "/etc/falco"
          }
        }
      }
    }
  }
}

resource "kubernetes_service" "falco_svc" {
  metadata {
    name = kubernetes_daemonset.falco_ds.metadata.0.name
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  spec {
    type = "ClusterIP"

    port {
      protocol = "TCP"
      port     = 8765
    }

    selector = {
      app  = "falco"
      role = "security"
    }
  }
}

caquino on 13 Jun 2019

❤2 🚀1

All 20 comments

I'm getting a similar issue on DigitalOcean Kubernetes Cluster nodes. Running on Debian.

* Setting up /usr/src links from host
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: * Mounting debugfs
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: Found kernel config at /host/boot/config-4.19.0-0.bpo.4-amd64
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: * Trying to compile BPF probe falco-probe-bpf (falco-probe-bpf-0.15.1-x86_64-4.19.0-0.bpo.4-amd64-1a4e8b8b750b65b8f458bd45cb424237.o)
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: make[1]: *** /lib/modules/4.19.0-0.bpo.4-amd64/build: No such file or directory.  Stop.
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: make: *** [Makefile:18: all] Error 2
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: * Trying to download precompiled BPF probe from https://s3.amazonaws.com/download.draios.com/stable/sysdig-probe-binaries/falco-probe-bpf-0.15.1-x86_64-4.19.0-0.bpo.4-amd64-1a4e8b8b750b65b8f458bd45cb424237.o
Jun 08 14:36:46 falco default[falco-daemonset-vkx27]: mv: cannot stat '/usr/src/falco-0.15.1/bpf/probe.o': No such file or directory
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: curl: (22) The requested URL returned error: 404 Not Found
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: * Failure to find a BPF probe
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: Sat Jun  8 12:36:46 2019: Falco initialized with configuration file /etc/falco/falco.yaml
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: Sat Jun  8 12:36:46 2019: Loading rules from file /etc/falco/falco_rules.yaml:
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: <11>1 2019-06-08T12:36:46Z  [] 1639 - - I0608 12:36:46.979726    1639 kubelet.go:1930] SyncLoop (PLEG): "falco-daemonset-vkx27_default(7ab164b6-89e8-11e9-bc26-1adc8e44b8e2)", event: &pleg.PodLifecycleEvent{ID:"7ab164b6-89e8-11e9-bc26-1adc8e44b8e2", Type:"ContainerStarted", Data:"e4484161b895ba7fe9b6f7c99b6132eb1d15f4827c1c2c4239cef53747ce538e"}
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: Sat Jun  8 12:36:47 2019: Loading rules from file /etc/falco/falco_rules.local.yaml:
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: Sat Jun  8 12:36:47 2019: Loading rules from file /etc/falco/k8s_audit_rules.yaml:
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: <11>1 2019-06-08T12:36:47Z  [] 1639 - - W0608 12:36:47.764673    1639 reflector.go:289] object-"kube-system"/"cilium-metrics-config": watch of *v1.ConfigMap ended with: too old resource version: 2303205 (2303703)
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: Sat Jun  8 12:36:47 2019: Unable to load the driver. Exiting.
Jun 08 14:36:47 falco default[falco-daemonset-vkx27]: Sat Jun  8 12:36:47 2019: Runtime error: can't open BPF probe '/root/.sysdig/falco-probe-bpf.o': No such file or directory. Exiting.

michiels on 8 Jun 2019

Hi @caquino thanks for opening this one.
Multiple users noticed the behavior, no one opened a specific issue about it that I'm aware of. This problem was caused because COS developers modified the audit kernel headers in a way that we cannot determine whether the loginuid field is available or not.

There's a fix in the BPF driver already but that had not been merged yet.

https://github.com/draios/sysdig/pull/1431

It would be helpful if you can test it in your environment and give feedback on how it works with Falco.

fntlnz on 8 Jun 2019

Hi @fntlnz more than happy to test it.

I was looking for the Dockerfile that I could use to build the cos-bpf branch but looks like the ones I found on the repository all rely on using packages from the draios repository, which I assume will not have a package for the branch.

Is there any direction you can point me on how to build the cos-bpf branch into a docker image that will run on COS?

caquino on 10 Jun 2019

draios/sysdig#1431 has been merged in

leodido on 11 Jun 2019

@caquino If you run this on your GKE cluster then let me know that it solves your problem. I believe it will (it solves it on my end), but unfortunately COS is a bit of a moving target so it will be helpful to hear it fixes your stuff as well :)

nathan-b on 11 Jun 2019

👀1 🚀1 ❤1 👍1

/area bpf-probe

fntlnz on 12 Jun 2019

@nathan-b do I need to use any specific image tag or just the latest should cover the PR that was merged?

The error now looks different, but it still fails sadly, this is what I get using the latest and dev tag images:

* Setting up /usr/src links from host
* Mounting debugfs
Found kernel config at /proc/config.gz
* COS detected (build 11647.182.0), downloading and setting up kernel headers
* Downloading https://storage.googleapis.com/cos-tools/11647.182.0/kernel-src.tar.gz
* Extracting kernel sources
* Configuring kernel
scripts/sign-file.c:25:30: fatal error: openssl/opensslv.h: No such file or directory
compilation terminated.
make[1]: *** [scripts/Makefile.host:102: scripts/sign-file] Error 1
make: *** [Makefile:572: scripts] Error 2
* Trying to compile BPF probe falco-probe-bpf (falco-probe-bpf-0.15.1-x86_64-4.14.119+-b2eece47169260af476d82b56b239305.o)
In file included from /usr/src/falco-0.15.1/bpf/probe.c:23:
/usr/src/falco-0.15.1/bpf/fillers.h:2017:26: error: no member named 'loginuid' in 'struct task_struct'
                loginuid = _READ(task->loginuid);
                                 ~~~~  ^
/usr/src/falco-0.15.1/bpf/plumbing_helpers.h:18:28: note: expanded from macro '_READ'
#define _READ(P) ({ typeof(P) _val;                             \
                           ^
In file included from /usr/src/falco-0.15.1/bpf/probe.c:23:
/usr/src/falco-0.15.1/bpf/fillers.h:2017:26: error: no member named 'loginuid' in 'struct task_struct'
                loginuid = _READ(task->loginuid);
                                 ~~~~  ^
/usr/src/falco-0.15.1/bpf/plumbing_helpers.h:20:44: note: expanded from macro '_READ'
                    bpf_probe_read(&_val, sizeof(_val), &P);    \
                                                         ^
In file included from /usr/src/falco-0.15.1/bpf/probe.c:23:
/usr/src/falco-0.15.1/bpf/fillers.h:2017:12: error: assigning to 'kuid_t' from incompatible type 'void'
                loginuid = _READ(task->loginuid);
                         ^ ~~~~~~~~~~~~~~~~~~~~~
3 errors generated.
make[2]: *** [/usr/src/falco-0.15.1/bpf/Makefile:33: /usr/src/falco-0.15.1/bpf/probe.o] Error 1
make[1]: *** [Makefile:1540: _module_/usr/src/falco-0.15.1/bpf] Error 2
make: *** [Makefile:18: all] Error 2
mv: cannot stat '/usr/src/falco-0.15.1/bpf/probe.o': No such file or directory
* Trying to download precompiled BPF probe from https://s3.amazonaws.com/download.draios.com/stable/sysdig-probe-binaries/falco-probe-bpf-0.15.1-x86_64-4.14.119%2B-b2eece47169260af476d82b56b239305.o
curl: (22) The requested URL returned error: 404 Not Found
* Failure to find a BPF probe
Wed Jun 12 16:20:54 2019: Falco initialized. No configuration file found, proceeding with defaults
Wed Jun 12 16:20:54 2019: Runtime error: You must specify at least one rules file/directory via -r or a rules_file entry in falco.yaml. Exiting.

caquino on 12 Jun 2019

We are cutting a new release to incorporate the fix from sysdig. Hopefully it will be out today.

mfdii on 12 Jun 2019

👍1

Ok, 0.15.3 has the fix in the eBPF probe so it will compile successfully. Unfortunately the COS version detection has a bug that prevents compile time flags from getting set.

If you set the environment variable in your daemonset like below

- name: KBUILD_EXTRA_CPPFLAGS
  value: -DCOS_73_WORKAROUND

the probe will build.

mfdii on 13 Jun 2019

@mfdii I can confirm that with the KBUILD_EXTRA_CPPFLAGS the build works without issues.

It also required some changes to my terraform to work, to automount the service account secret.

I'm sharing the final configuration here in case anyone else is interested.

resource "kubernetes_service_account" "falco_sa" {
  metadata {
    name = "falco-account"
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  automount_service_account_token = true
}

resource "kubernetes_cluster_role" "falco_cr" {
  metadata {
    name = "falco-cluster-role"
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  rule {
    api_groups = ["extensions", ""]
    resources  = ["nodes", "namespaces", "pods", "replicationcontrollers", "replicasets", "services", "daemonsets", "deployments", "events", "configmaps"]
    verbs      = ["get", "list", "watch"]
  }
  rule {
    non_resource_urls = ["/healthz", "/healthz/*"]
    verbs             = ["get"]
  }
}

resource "kubernetes_cluster_role_binding" "falco_crb" {
  metadata {
    name = "falco-cluster-role-bind"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  subject {
    kind      = "ServiceAccount"
    name      = kubernetes_service_account.falco_sa.metadata.0.name
    namespace = "default"
  }

  role_ref {
    kind      = "ClusterRole"
    name      = kubernetes_cluster_role.falco_cr.metadata.0.name
    api_group = "rbac.authorization.k8s.io"
  }
}

resource "kubernetes_config_map" "falco_cfgmap" {
  metadata {
    name = "falco-cfgmap"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  data = {
    "application_rules.yaml" = file("configs/falco/application_rules.yaml")
    "falco_rules.local.yaml" = file("configs/falco/falco_rules.local.yaml")
    "falco_rules.yaml"       = file("configs/falco/falco_rules.yaml")
    "k8s_audit_rules.yaml"   = file("configs/falco/k8s_audit_rules.yaml")
    "falco.yaml"             = file("configs/falco/falco.yaml")
  }
}

resource "kubernetes_daemonset" "falco_ds" {
  metadata {
    name = "falco-daemonset"
    labels = {
      app  = "falco"
      role = "security"
    }
  }

  spec {

    selector {
      match_labels = {
        app  = "falco"
        role = "security"
      }
    }

    template {
      metadata {
        labels = {
          app  = "falco"
          role = "security"
        }
      }

      spec {
        host_network         = true
        service_account_name = kubernetes_service_account.falco_sa.metadata.0.name
        dns_policy           = "ClusterFirstWithHostNet"

        volume {
          name = "docker-socket"
          host_path {
            path = "/var/run/docker.socket"
          }
        }
        volume {
          name = "containerd-socket"
          host_path {
            path = "/run/containerd/containerd.sock"
          }
        }
        volume {
          name = "dev-fs"
          host_path {
            path = "/dev"
          }
        }
        volume {
          name = "proc-fs"
          host_path {
            path = "/proc"
          }
        }
        volume {
          name = "boot-fs"
          host_path {
            path = "/boot"
          }
        }
        volume {
          name = "lib-modules"
          host_path {
            path = "/lib/modules"
          }
        }
        volume {
          name = "usr-fs"
          host_path {
            path = "/usr"
          }
        }
        volume {
          name = "etc-fs"
          host_path {
            path = "/etc"
          }
        }
        volume {
          name = "dshm"
          empty_dir {
            medium = "Memory"
          }
        }
        volume {
          name = "falco-config"
          config_map {
            name = kubernetes_config_map.falco_cfgmap.metadata.0.name
          }
        }

        container {
          name  = "falco"
          image = "falcosecurity/falco:latest"
          args = [
            "/usr/bin/falco",
            "--cri", "/host/run/containerd/containerd.sock",
            "-K", "/var/run/secrets/kubernetes.io/serviceaccount/token",
            "-k", "https://$(KUBERNETES_SERVICE_HOST)",
            "-pk",
          ]
          security_context {
            privileged = true
          }
          env {
            name  = "SYSDIG_BPF_PROBE"
            value = ""
          }
          env {
            name  = "KBUILD_EXTRA_CPPFLAGS"
            value = "-DCOS_73_WORKAROUND"
          }
          volume_mount {
            name       = "docker-socket"
            mount_path = "/host/var/run/docker.sock"
          }
          volume_mount {
            name       = "containerd-socket"
            mount_path = "/host/run/containerd/containerd.sock"
          }
          volume_mount {
            name       = "dev-fs"
            mount_path = "/host/dev"
          }
          volume_mount {
            name       = "proc-fs"
            mount_path = "/host/proc"
            read_only  = true
          }
          volume_mount {
            name       = "boot-fs"
            mount_path = "/host/boot"
            read_only  = true
          }
          volume_mount {
            name       = "lib-modules"
            mount_path = "/host/lib/modules"
            read_only  = true
          }
          volume_mount {
            name       = "usr-fs"
            mount_path = "/host/usr"
            read_only  = true
          }
          volume_mount {
            name       = "etc-fs"
            mount_path = "/host/etc"
            read_only  = true
          }
          volume_mount {
            name       = "dshm"
            mount_path = "/dev/shm"
          }
          volume_mount {
            name       = "falco-config"
            mount_path = "/etc/falco"
          }
        }
      }
    }
  }
}

resource "kubernetes_service" "falco_svc" {
  metadata {
    name = kubernetes_daemonset.falco_ds.metadata.0.name
    labels = {
      app  = "falco"
      role = "security"
    }
  }
  spec {
    type = "ClusterIP"

    port {
      protocol = "TCP"
      port     = 8765
    }

    selector = {
      app  = "falco"
      role = "security"
    }
  }
}

caquino on 13 Jun 2019

❤2 🚀1

Thanks @caquino thats solid feedback !

You terraform config seems a perfect candidate for a documentation page!

Closing this as it seems solved! Please reopen if needed !

fntlnz on 13 Jun 2019

/close

fntlnz on 13 Jun 2019

@fntlnz: Closing this issue.

In response to this:

/close

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes/test-infra repository.

poiana on 13 Jun 2019

/kind bug

fntlnz on 13 Jun 2019

@mfdii Any idea how to inject that environment variable with helm?

Envvar:

- name: KBUILD_EXTRA_CPPFLAGS
  value: -DCOS_73_WORKAROUND

hus787 on 2 Jul 2019

Unfortunately not, you need to download the helm chart and add the env
variable to the daemonset template.

On Tue, Jul 2, 2019 at 11:53 AM Hussain Parsaiyan notifications@github.com
wrote:

@mfdii https://github.com/mfdii Any idea how to inject that environment
variable with helm?

Envvar:

name: KBUILD_EXTRA_CPPFLAGS
value: -DCOS_73_WORKAROUND

—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub
https://github.com/falcosecurity/falco/issues/650?email_source=notifications&email_token=AAM74DMQX2554YCX6U3SULTP5N2Y7A5CNFSM4HUPOHVKYY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGODZBXN4A#issuecomment-507737840,
or mute the thread
https://github.com/notifications/unsubscribe-auth/AAM74DIBEZB6I3CB6OAYT4TP5N2Y7ANCNFSM4HUPOHVA
.

mfdii on 2 Jul 2019

Hi folks. I was unable to get this to work on my GKE cluster until I switched the image tag from 0.15.3 to dev. Prior to that, I was still seeing the compilation issue re: loginuid. I had been (and still am) applying COS_73_WORKAROUND, so I'm not sure what changed between 0.15.3 and dev, but whatever it was, it made falco work for me!

zhaytee on 3 Jul 2019

❤1

@mfdii It isn't working for coreos-stable-2135.5.0 release, tried will all the falco versions above 0.13.1.

make[4]: *** [/usr/src/falco-0.1.2811dev/bpf/Makefile:33: /usr/src/falco-0.1.2811dev/bpf/probe.o] Error 1
make[3]: *** [/host/lib/modules/4.19.50-coreos-r1/source/Makefile:1518: _module_/usr/src/falco-0.1.2811dev/bpf] Error 2
make[2]: *** [Makefile:146: sub-make] Error 2
make[1]: *** [Makefile:24: __sub-make] Error 2
make: *** [Makefile:18: all] Error 2
mv: cannot stat '/usr/src/falco-0.1.2811dev/bpf/probe.o': No such file or directory
* Trying to download precompiled BPF probe from https://s3.amazonaws.com/download.draios.com/stable/sysdig-probe-binaries/falco-probe-bpf-0.1.2811dev-x86_64-4.19.50-coreos-r1-4c309a0cf89dae55a9a9cc0f602f2fe1.o
curl: (22) The requested URL returned error: 404 Not Found
* Failure to find a BPF probe
Tue Aug  6 10:56:40 2019: Falco initialized with configuration file /etc/falco/falco.yaml
Tue Aug  6 10:56:40 2019: Loading rules from file /etc/falco/falco_rules.yaml:
Tue Aug  6 10:56:40 2019: Loading rules from file /etc/falco/falco_rules.local.yaml:
Tue Aug  6 10:56:40 2019: Loading rules from file /etc/falco/application_rules.yaml:
Tue Aug  6 10:56:40 2019: Unable to load the driver. Exiting.
Tue Aug  6 10:56:40 2019: Runtime error: can't open BPF probe '/root/.sysdig/falco-probe-bpf.o': No such file or directory. Exiting.

ankitjain28may on 6 Aug 2019

CoreOS is not supported with the eBPF probe. You have to use the kernel module for CoreOS

mfdii on 7 Aug 2019

@mfdii On using kernel modules, it's shutting down the instance due to kernel panic error.
Showing module signature verification failed, probe not found, kernel panic, system reboot.

ankitjain28may on 8 Aug 2019

👍1

Was this page helpful?

0 / 5 - 0 ratings