Datadog-agent: DD_APM_IGNORE_RESOURCES is not considering the reported values.

[ralmsdeveloper]

Original from @https://github.com/DataDog/dd-trace-dotnet/issues/896

First sorry if this is not the best place!

But I'm having problems using DD_APM_IGNORE_RESOURCES I can't help it sending a flurry of things to the datadog.

I am leaving a small repo of my container below.

I would like the agent to disregard a particular route.
GET /healthcheck and GET healthcheck

I'm running ASPNET Core 3.1

FROM mcr.microsoft.com/dotnet/core/aspnet:3.1 AS base
WORKDIR /app

COPY ./publish /app

RUN curl -LO https://github.com/DataDog/dd-trace-dotnet/releases/download/v1.19.1/datadog-dotnet-apm_1.19.1_amd64.deb \
&& dpkg -i ./datadog-dotnet-apm_1.19.1_amd64.deb 

ENV ASPNETCORE_ENVIRONMENT= ..OTHER_VARIABLES..
DD_APM_IGNORE_RESOURCES="GET healthcheck"

[ralmsdeveloper]

Solved!

[shoira-shamsieva]

@ralmsdeveloper how did you solve it?

[dulago]

@ralmsdeveloper could you share your solution please? I'm running through this very same issue with healthchecks flooding datadog

[ralmsdeveloper]

Sorry to all for the delay in the response!

Basically, what I did was add the variable(DD_APM_IGNORE_RESOURCES) to my yaml file.

Sample:

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: datadog-agent
  namespace: default
  labels: {}
spec:
  selector:
    matchLabels:
      app: datadog-agent
  template:
    metadata:
      labels:
        app: datadog-agent
      name: datadog-agent
      annotations:
        container.apparmor.security.beta.kubernetes.io/system-probe: unconfined
        container.seccomp.security.alpha.kubernetes.io/system-probe: localhost/system-probe
    spec:
      containers:
        - name: agent
          image: "datadog/agent:7.23.1"
          imagePullPolicy: IfNotPresent
          command: ["agent", "run"]
          resources: {}
          ports:
            - containerPort: 8125
              name: dogstatsdport
              protocol: UDP
          env:
            - name: DD_API_KEY
              valueFrom:
                secretKeyRef:
                  name: "datadog-agent"
                  key: api-key
            - name: DD_KUBERNETES_KUBELET_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: KUBERNETES
              value: "yes"
            - name: DOCKER_HOST
              value: unix:///host/var/run/docker.sock
            - name: DD_LOG_LEVEL
              value: "INFO"
            - name: DD_DOGSTATSD_PORT
              value: "8125"
            - name: DD_APM_ENABLED
              value: "false"
            - name: DD_LOGS_ENABLED
              value: "true"
            - name: DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL
              value: "true"
            - name: DD_LOGS_CONFIG_K8S_CONTAINER_USE_FILE
              value: "true"
            - name: DD_HEALTH_PORT
              value: "5555"
            - name: DD_APM_IGNORE_RESOURCES
              value: GET /healthCheck,getLastError admin query
          volumeMounts:
            - name: config
              mountPath: /etc/datadog-agent
            - name: runtimesocketdir
              mountPath: /host/var/run
              mountPropagation: None
              readOnly: true
            - name: sysprobe-socket-dir
              mountPath: /var/run/sysprobe
              readOnly: true
            - name: sysprobe-config
              mountPath: /etc/datadog-agent/system-probe.yaml
              subPath: system-probe.yaml
            - name: procdir
              mountPath: /host/proc
              mountPropagation: None
              readOnly: true
            - name: cgroups
              mountPath: /host/sys/fs/cgroup
              mountPropagation: None
              readOnly: true
            - name: pointerdir
              mountPath: /opt/datadog-agent/run
              mountPropagation: None
            - name: logpodpath
              mountPath: /var/log/pods
              mountPropagation: None
              readOnly: true
            - name: logdockercontainerpath
              mountPath: /var/lib/docker/containers
              mountPropagation: None
              readOnly: true
          livenessProbe:
            failureThreshold: 6
            httpGet:
              path: /health
              port: 5555
              scheme: HTTP
            initialDelaySeconds: 15
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 5
          readinessProbe:
            failureThreshold: 6
            httpGet:
              path: /health
              port: 5555
              scheme: HTTP
            initialDelaySeconds: 15
            periodSeconds: 15
            successThreshold: 1
            timeoutSeconds: 5
        - name: trace-agent
          image: "datadog/agent:7.23.1"
          imagePullPolicy: IfNotPresent
          command: ["trace-agent", "-config=/etc/datadog-agent/datadog.yaml"]
          resources: {}
          ports:
            - containerPort: 8126
              hostPort: 8126
              name: traceport
              protocol: TCP
          env:
            - name: DD_API_KEY
              valueFrom:
                secretKeyRef:
                  name: "datadog-agent"
                  key: api-key
            - name: DD_KUBERNETES_KUBELET_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: KUBERNETES
              value: "yes"
            - name: DOCKER_HOST
              value: unix:///host/var/run/docker.sock
            - name: DD_LOG_LEVEL
              value: "INFO"
            - name: DD_APM_ENABLED
              value: "true"
            - name: DD_APM_NON_LOCAL_TRAFFIC
              value: "true"
            - name: DD_APM_RECEIVER_PORT
              value: "8126"
            - name: DD_APM_IGNORE_RESOURCES
              value: GET /healthCheck,getLastError admin query           
          volumeMounts:
            - name: config
              mountPath: /etc/datadog-agent
            - name: runtimesocketdir
              mountPath: /host/var/run
              mountPropagation: None
              readOnly: true
          livenessProbe:
            initialDelaySeconds: 15
            periodSeconds: 15
            tcpSocket:
              port: 8126
            timeoutSeconds: 5
        - name: process-agent
          image: "datadog/agent:7.23.1"
          imagePullPolicy: IfNotPresent
          command: ["process-agent", "-config=/etc/datadog-agent/datadog.yaml"]
          resources: {}
          env:
            - name: DD_API_KEY
              valueFrom:
                secretKeyRef:
                  name: "datadog-agent"
                  key: api-key
            - name: DD_KUBERNETES_KUBELET_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: KUBERNETES
              value: "yes"
            - name: DOCKER_HOST
              value: unix:///host/var/run/docker.sock
            - name: DD_PROCESS_AGENT_ENABLED
              value: "true"
            - name: DD_LOG_LEVEL
              value: "INFO"
            - name: DD_SYSTEM_PROBE_ENABLED
              value: "false"
            - name: DD_ORCHESTRATOR_EXPLORER_ENABLED
              value: "false"
          volumeMounts:
            - name: config
              mountPath: /etc/datadog-agent
            - name: runtimesocketdir
              mountPath: /host/var/run
              mountPropagation: None
              readOnly: true
            - name: cgroups
              mountPath: /host/sys/fs/cgroup
              mountPropagation: None
              readOnly: true
            - name: passwd
              mountPath: /etc/passwd
            - name: procdir
              mountPath: /host/proc
              mountPropagation: None
              readOnly: true
            - name: sysprobe-socket-dir
              mountPath: /var/run/sysprobe
              readOnly: true
            - name: sysprobe-config
              mountPath: /etc/datadog-agent/system-probe.yaml
              subPath: system-probe.yaml
        - name: system-probe
          image: "datadog/agent:7.23.1"
          imagePullPolicy: IfNotPresent
          securityContext:
            capabilities:
              add: ["SYS_ADMIN", "SYS_RESOURCE", "SYS_PTRACE", "NET_ADMIN", "NET_BROADCAST", "IPC_LOCK"]
          command: ["/opt/datadog-agent/embedded/bin/system-probe", "--config=/etc/datadog-agent/system-probe.yaml"]
          env:
            - name: DD_API_KEY
              valueFrom:
                secretKeyRef:
                  name: "datadog-agent"
                  key: api-key
            - name: DD_KUBERNETES_KUBELET_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: KUBERNETES
              value: "yes"
            - name: DOCKER_HOST
              value: unix:///host/var/run/docker.sock
            - name: DD_LOG_LEVEL
              value: "INFO"
          resources: {}
          volumeMounts:
            - name: debugfs
              mountPath: /sys/kernel/debug
              mountPropagation: None
            - name: sysprobe-config
              mountPath: /etc/datadog-agent/system-probe.yaml
              subPath: system-probe.yaml
            - name: sysprobe-socket-dir
              mountPath: /var/run/sysprobe
            - name: procdir
              mountPath: /host/proc
              mountPropagation: None
              readOnly: true
      initContainers:
        - name: init-volume
          image: "datadog/agent:7.23.1"
          imagePullPolicy: IfNotPresent
          command: ["bash", "-c"]
          args:
            - cp -r /etc/datadog-agent /opt
          volumeMounts:
            - name: config
              mountPath: /opt/datadog-agent
          resources: {}
        - name: init-config
          image: "datadog/agent:7.23.1"
          imagePullPolicy: IfNotPresent
          command: ["bash", "-c"]
          args:
            - for script in $(find /etc/cont-init.d/ -type f -name '*.sh' | sort) ; do bash $script ; done
          volumeMounts:
            - name: config
              mountPath: /etc/datadog-agent
            - name: procdir
              mountPath: /host/proc
              mountPropagation: None
              readOnly: true
            - name: runtimesocketdir
              mountPath: /host/var/run
              mountPropagation: None
              readOnly: true
            - name: sysprobe-config
              mountPath: /etc/datadog-agent/system-probe.yaml
              subPath: system-probe.yaml
          env:
            - name: DD_API_KEY
              valueFrom:
                secretKeyRef:
                  name: "datadog-agent"
                  key: api-key
            - name: DD_KUBERNETES_KUBELET_HOST
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
            - name: KUBERNETES
              value: "yes"
            - name: DOCKER_HOST
              value: unix:///host/var/run/docker.sock
          resources: {}
        - name: seccomp-setup
          image: "datadog/agent:7.23.1"
          command:
            - cp
            - /etc/config/system-probe-seccomp.json
            - /host/var/lib/kubelet/seccomp/system-probe
          volumeMounts:
            - name: datadog-agent-security
              mountPath: /etc/config
            - name: seccomp-root
              mountPath: /host/var/lib/kubelet/seccomp
              mountPropagation: None
          resources: {}
      volumes:
        - name: config
          emptyDir: {}
        - hostPath:
            path: /var/run
          name: runtimesocketdir
        - hostPath:
            path: /proc
          name: procdir
        - hostPath:
            path: /sys/fs/cgroup
          name: cgroups
        - name: s6-run
          emptyDir: {}
        - name: sysprobe-config
          configMap:
            name: datadog-agent-system-probe-config
        - name: datadog-agent-security
          configMap:
            name: datadog-agent-security
        - hostPath:
            path: /var/lib/kubelet/seccomp
          name: seccomp-root
        - hostPath:
            path: /sys/kernel/debug
          name: debugfs
        - name: sysprobe-socket-dir
          emptyDir: {}
        - hostPath:
            path: /etc/passwd
          name: passwd
        - hostPath:
            path: "/var/lib/datadog-agent/logs"
          name: pointerdir
        - hostPath:
            path: /var/log/pods
          name: logpodpath
        - hostPath:
            path: /var/lib/docker/containers
          name: logdockercontainerpath
      tolerations:
      affinity: {}
      serviceAccountName: "datadog-agent"
      nodeSelector:
        kubernetes.io/os: linux
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 10%
    type: RollingUpdate