Kibana: [SIEM] [Maps] Network Map fails to load data with failed request to '/internal/search/es'

Created on 2 Apr 2020 · 11Comments · Source: elastic/kibana

In testing https://github.com/elastic/kibana/pull/61165, it was noticed that the SIEM Network Map (Map Embeddable) was failing to load data. The same behavior was then verified against master (e202fe7aa31721a4c319aa414cc4e6739b9bf000), albeit slightly different (sometimes returning a 403 instead of 400).

This can be verified internally by on siem-dev here: https://kibana.siem.estc.dev/app/siem#/network/flows

`/internal/search/es` -- `400` (consistent)

Request paylod

{
  "params": {
    "ignoreThrottled": true,
    "preference": 1585846087508,
    "index": "auditbeat-*",
    "body": {
      "docvalue_fields": ["source.geo.location"],
      "size": 10000,
      "_source": false,
      "stored_fields": ["source.geo.location"],
      "script_fields": {},
      "query": {
        "bool": {
          "must": [],
          "filter": [
            { "match_all": {} },
            { "match_all": {} },
            {
              "range": {
                "@timestamp": {
                  "gte": "2020-04-02T16:34:32.538Z",
                  "lte": "2020-04-02T16:49:32.538Z",
                  "format": "strict_date_optional_time"
                }
              }
            }
          ],
          "should": [],
          "must_not": []
        }
      }
    },
    "rest_total_hits_as_int": true,
    "ignore_unavailable": true,
    "ignore_throttled": true,
    "timeout": "30000ms"
  },
  "serverStrategy": "es"
}

Response payload

{
  "statusCode": 400,
  "error": "Bad Request",
  "message": "Bad Request",
  "attributes": { "error": "Bad Request" }
}

`/internal/search/es` -- `403` (sporadic)

Request payload

{
  "params": {
    "ignoreThrottled": true,
    "preference": 1585849411730,
    "index": "filebeat-*",
    "body": {
      "size": 0,
      "aggs": {
        "destSplit": {
          "terms": {
            "script": {
              "source": "doc['destination.geo.location'].value.toString()",
              "lang": "painless"
            },
            "order": { "_count": "desc" },
            "size": 100
          },
          "aggs": {
            "sourceGrid": {
              "geotile_grid": {
                "field": "source.geo.location",
                "precision": 6,
                "size": 500
              },
              "aggs": {
                "sourceCentroid": {
                  "geo_centroid": { "field": "source.geo.location" }
                },
                "sum_of_source.bytes": { "sum": { "field": "source.bytes" } },
                "sum_of_destination.bytes": {
                  "sum": { "field": "destination.bytes" }
                }
              }
            }
          }
        }
      },
      "stored_fields": ["*"],
      "script_fields": {},
      "docvalue_fields": [
        { "field": "@timestamp", "format": "date_time" },
        {
          "field": "azure.auditlogs.properties.activity_datetime",
          "format": "date_time"
        },
        { "field": "azure.enqueued_time", "format": "date_time" },
        { "field": "cef.extensions.agentReceiptTime", "format": "date_time" },
        { "field": "cef.extensions.deviceCustomDate1", "format": "date_time" },
        { "field": "cef.extensions.deviceCustomDate2", "format": "date_time" },
        { "field": "cef.extensions.deviceReceiptTime", "format": "date_time" },
        { "field": "cef.extensions.endTime", "format": "date_time" },
        { "field": "cef.extensions.fileCreateTime", "format": "date_time" },
        {
          "field": "cef.extensions.fileModificationTime",
          "format": "date_time"
        },
        { "field": "cef.extensions.flexDate1", "format": "date_time" },
        { "field": "cef.extensions.managerReceiptTime", "format": "date_time" },
        { "field": "cef.extensions.oldFileCreateTime", "format": "date_time" },
        {
          "field": "cef.extensions.oldFileModificationTime",
          "format": "date_time"
        },
        { "field": "cef.extensions.startTime", "format": "date_time" },
        { "field": "event.created", "format": "date_time" },
        { "field": "event.end", "format": "date_time" },
        { "field": "event.ingested", "format": "date_time" },
        { "field": "event.start", "format": "date_time" },
        { "field": "file.accessed", "format": "date_time" },
        { "field": "file.created", "format": "date_time" },
        { "field": "file.ctime", "format": "date_time" },
        { "field": "file.mtime", "format": "date_time" },
        { "field": "kafka.block_timestamp", "format": "date_time" },
        { "field": "misp.campaign.first_seen", "format": "date_time" },
        { "field": "misp.campaign.last_seen", "format": "date_time" },
        { "field": "misp.intrusion_set.first_seen", "format": "date_time" },
        { "field": "misp.intrusion_set.last_seen", "format": "date_time" },
        { "field": "misp.observed_data.first_observed", "format": "date_time" },
        { "field": "misp.observed_data.last_observed", "format": "date_time" },
        { "field": "misp.report.published", "format": "date_time" },
        { "field": "misp.threat_indicator.valid_from", "format": "date_time" },
        { "field": "misp.threat_indicator.valid_until", "format": "date_time" },
        {
          "field": "netflow.collection_time_milliseconds",
          "format": "date_time"
        },
        { "field": "netflow.flow_end_microseconds", "format": "date_time" },
        { "field": "netflow.flow_end_milliseconds", "format": "date_time" },
        { "field": "netflow.flow_end_nanoseconds", "format": "date_time" },
        { "field": "netflow.flow_end_seconds", "format": "date_time" },
        { "field": "netflow.flow_start_microseconds", "format": "date_time" },
        { "field": "netflow.flow_start_milliseconds", "format": "date_time" },
        { "field": "netflow.flow_start_nanoseconds", "format": "date_time" },
        { "field": "netflow.flow_start_seconds", "format": "date_time" },
        { "field": "netflow.max_export_seconds", "format": "date_time" },
        { "field": "netflow.max_flow_end_microseconds", "format": "date_time" },
        { "field": "netflow.max_flow_end_milliseconds", "format": "date_time" },
        { "field": "netflow.max_flow_end_nanoseconds", "format": "date_time" },
        { "field": "netflow.max_flow_end_seconds", "format": "date_time" },
        { "field": "netflow.min_export_seconds", "format": "date_time" },
        {
          "field": "netflow.min_flow_start_microseconds",
          "format": "date_time"
        },
        {
          "field": "netflow.min_flow_start_milliseconds",
          "format": "date_time"
        },
        {
          "field": "netflow.min_flow_start_nanoseconds",
          "format": "date_time"
        },
        { "field": "netflow.min_flow_start_seconds", "format": "date_time" },
        {
          "field": "netflow.monitoring_interval_end_milli_seconds",
          "format": "date_time"
        },
        {
          "field": "netflow.monitoring_interval_start_milli_seconds",
          "format": "date_time"
        },
        {
          "field": "netflow.observation_time_microseconds",
          "format": "date_time"
        },
        {
          "field": "netflow.observation_time_milliseconds",
          "format": "date_time"
        },
        {
          "field": "netflow.observation_time_nanoseconds",
          "format": "date_time"
        },
        { "field": "netflow.observation_time_seconds", "format": "date_time" },
        {
          "field": "netflow.system_init_time_milliseconds",
          "format": "date_time"
        },
        { "field": "package.installed", "format": "date_time" },
        { "field": "process.parent.start", "format": "date_time" },
        { "field": "process.start", "format": "date_time" },
        { "field": "suricata.eve.flow.end", "format": "date_time" },
        { "field": "suricata.eve.flow.start", "format": "date_time" },
        { "field": "suricata.eve.timestamp", "format": "date_time" },
        { "field": "suricata.eve.tls.notafter", "format": "date_time" },
        { "field": "suricata.eve.tls.notbefore", "format": "date_time" },
        { "field": "tls.client.not_after", "format": "date_time" },
        { "field": "tls.client.not_before", "format": "date_time" },
        { "field": "tls.server.not_after", "format": "date_time" },
        { "field": "tls.server.not_before", "format": "date_time" },
        { "field": "zeek.kerberos.valid.from", "format": "date_time" },
        { "field": "zeek.kerberos.valid.until", "format": "date_time" },
        { "field": "zeek.ocsp.revoke.time", "format": "date_time" },
        { "field": "zeek.ocsp.update.next", "format": "date_time" },
        { "field": "zeek.ocsp.update.this", "format": "date_time" },
        { "field": "zeek.pe.compile_time", "format": "date_time" },
        { "field": "zeek.smb_files.times.accessed", "format": "date_time" },
        { "field": "zeek.smb_files.times.changed", "format": "date_time" },
        { "field": "zeek.smb_files.times.created", "format": "date_time" },
        { "field": "zeek.smb_files.times.modified", "format": "date_time" },
        { "field": "zeek.smtp.date", "format": "date_time" },
        { "field": "zeek.snmp.up_since", "format": "date_time" },
        { "field": "zeek.x509.certificate.valid.from", "format": "date_time" },
        { "field": "zeek.x509.certificate.valid.until", "format": "date_time" }
      ],
      "_source": { "excludes": [] },
      "query": {
        "bool": {
          "must": [],
          "filter": [
            { "match_all": {} },
            { "match_all": {} },
            {
              "geo_bounding_box": {
                "destination.geo.location": {
                  "top_left": [-140.625, 48.9225],
                  "bottom_right": [-28.125, 21.94305]
                }
              }
            },
            {
              "range": {
                "@timestamp": {
                  "gte": "2020-04-01T17:43:34.626Z",
                  "lte": "2020-04-02T17:43:34.626Z",
                  "format": "strict_date_optional_time"
                }
              }
            }
          ],
          "should": [],
          "must_not": []
        }
      }
    },
    "rest_total_hits_as_int": true,
    "ignore_unavailable": true,
    "ignore_throttled": true,
    "timeout": "30000ms"
  },
  "serverStrategy": "es"
}

Response payload

{
  "statusCode": 403,
  "error": "Forbidden",
  "message": "[security_exception] action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]",
  "attributes": {
    "error": {
      "root_cause": [
        {
          "type": "security_exception",
          "reason": "action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]"
        }
      ],
      "type": "security_exception",
      "reason": "action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]"
    }
  }
}

PR sent SecuritySolution AppServices Geo SIEM bug regression v7.7.0 v7.8.0 v8.0.0

Source

spong

Most helpful comment

I opened https://github.com/elastic/elasticsearch/pull/54761 for the 403 that we're seeing. This happens when the .async-search index is stored on a different node than the node that executes the search. This explains the unauthorized error (403) that is returned here but not the 400 (bad request).
I am not able to reproduce the latter so I have no idea where they're coming from.

jimczi on 4 Apr 2020

👍2

All 11 comments

Pinging @elastic/kibana-gis (Team:Geo)

elasticmachine on 2 Apr 2020

Pinging @elastic/siem (Team:SIEM)

elasticmachine on 2 Apr 2020

Pinging @elastic/kibana-app-arch (Team:AppArch)

elasticmachine on 2 Apr 2020

"message": "[security_exception] action [indices:data/write/bulk[s]] is unauthorized for user [_async_search]",

FWIW I've been seeing this on Discover too (on that same cluster spong mentioned). But I don't know what's causing it.

andrewkroh on 2 Apr 2020

cc @lukasolson and @lizozom in case this is related to search strategies.

stacey-gammon on 2 Apr 2020

Can we get a list of the roles/privileges the user that is being logged into has?

lukasolson on 2 Apr 2020

My test user has the out of the box superuser role on this cluster:

spong on 2 Apr 2020

EDIT: not 100% sure if this is the same issue, but exhibiting similar behavior.

This is on elastic cloud.

Hey y'all, I think this is effecting 7.7.0 as well. I just tried to visualize one of the pre-canned maps (In the maps app directly). It failed in a similar manner.

I tried the query manually via dev console and it worked fine. Both with _search and _async_search.

ES build info:

"build" : {
    "hash" : "54915a16830751ed38330b14023fc54ee1770c92",
    "date" : "2020-04-02T09:30:34.501251Z"
  },

Kibana Build:
https://github.com/elastic/kibana/commits/866dc65

Message in response body

{"statusCode":400,"error":"Bad Request","message":"Bad Request","attributes":{"error":"Bad Request"}}

Opened new issue as this seems fairly wide spread: https://github.com/elastic/kibana/issues/62502

benwtrent on 3 Apr 2020

👍1

Just deployed a fresh 7.7.0-BC4 on Elastic Cloud and am seeing the same behavior as @benwtrent.

Reproducible on the SIEM Network Map:

And when creating a map within the Maps app as well:

spong on 3 Apr 2020

FYI, the request that is sent to Elasticsearch looks something like this:

POST {index}/_async_search?wait_for_completion_timeout=1s&track_total_hits=true&ignore_unavailable=true&ignore_throttled=true&preference=1585956064575&rest_total_hits_as_int=true
{
  "version": true,
  "size": 500,
  "sort": [
    {
      "@timestamp": {
        "order": "desc",
        "unmapped_type": "boolean"
      }
    }
  ],
  "aggs": {
    "2": {
      "date_histogram": {
        "field": "@timestamp",
        "fixed_interval": "30s",
        "time_zone": "America/Phoenix",
        "min_doc_count": 1
      }
    }
  },
  "stored_fields": [
    "*"
  ],
  "script_fields": {},
  "docvalue_fields": [
    {
      "field": "@timestamp",
      "format": "date_time"
    }
  ],
  "_source": {
    "excludes": []
  },
  "query": {
    "bool": {
      "must": [],
      "filter": [
        {
          "match_all": {}
        },
        {
          "range": {
            "@timestamp": {
              "gte": "2020-04-03T23:06:13.394Z",
              "lte": "2020-04-03T23:21:13.394Z",
              "format": "strict_date_optional_time"
            }
          }
        }
      ],
      "should": [],
      "must_not": []
    }
  },
  "highlight": {
    "pre_tags": [
      "@kibana-highlighted-field@"
    ],
    "post_tags": [
      "@/kibana-highlighted-field@"
    ],
    "fields": {
      "*": {}
    },
    "fragment_size": 2147483647
  }
}

lukasolson on 4 Apr 2020

👍1

jimczi on 4 Apr 2020

👍2

Was this page helpful?

0 / 5 - 0 ratings

Related issues

Embedded objects in a dashboard should use the same font across different OSs

stacey-gammon · 3Comments

Discover carets are all different sizes

snide · 3Comments

No "auto" choice for Interval

timmolter · 3Comments

Use the visualizeLoader for dashboards

timroes · 3Comments

Emoji Support

cafuego · 3Comments

Kibana: [SIEM] [Maps] Network Map fails to load data with failed request to '/internal/search/es'

/internal/search/es -- 400 (consistent)

/internal/search/es -- 403 (sporadic)

Most helpful comment

All 11 comments

Related issues

`/internal/search/es` -- `400` (consistent)

`/internal/search/es` -- `403` (sporadic)