Kibana: Kibana 4 - Courier Fetch: X of 5 shards failed
Hi,
I'm having issues viewing Netflow data using Kibana 4.0.2 (Build 6004). The error message I get when I click the Discover tab is a yellow banner that reads:
Courier Fetch: 5 of 5 shards failed.
I am however able to view this data using Kibana 3, which leads me to believe this is a bug (or a change) in how Kibana 4 queries Elasticsearch.
I should also mention that I can view this data in Kibana 4 if I let Elasticsearch create a mapping dynamically for this index, but I would like to use a custom one so that it's optimized. I've included the mapping I'm trying to use below, and I don't see anything wrong with it. Netflow data includes bigger numbers than "type": "long" can handle, so I needed to use "type": "string" for some fields.
Request
curl -XPOST http://fqdn.omitted.com:5601/elasticsearch/_msearch?timeout=0&ignore_unavailable=true&preference=1432705287095
Request Payload
{
"index": "customindex-*",
"ignore_unavailable": true
}\n
{
"size": 500,
"sort": {
"@timestamp": "desc"
},
"highlight": {
"pre_tags": [
"@kibana-highlighted-field@"
],
"post_tags": [
"@/kibana-highlighted-field@"
],
"fields": {
"*": {}
}
},
"aggs": {
"2": {
"date_histogram": {
"field": "@timestamp",
"interval": "30s",
"pre_zone": "-07:00",
"pre_zone_adjust_large_interval": true,
"min_doc_count": 0,
"extended_bounds": {
"min": 1432704390127,
"max": 1432705290128
}
}
}
},
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter": {
"bool": {
"must": [
{
"range": {
"@timestamp": {
"gte": 1432704390134,
"lte": 1432705290134
}
}
}
],
"must_not": []
}
}
}
},
"fields": [
"*",
"_source"
],
"script_fields": {},
"fielddata_fields": [
"@timestamp"
]
}
Response
{
"responses": [
{
"took": 44,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 0,
"failed": 5,
"failures": [
{
"index": "customindex-2015.05.27",
"shard": 0,
"status": 500,
"reason": "RemoteTransportException[[fqdn.omitted.com][inet[/192.168.1.110:9300]][indices:data/read/search[phase/fetch/id]]]; nested: ElasticsearchIllegalStateException[No matching token for number_type [BIG_INTEGER]]; "
},
{
"index": "customindex-2015.05.27",
"shard": 1,
"status": 500,
"reason": "RemoteTransportException[[fqdn.omitted.com][inet[/192.168.1.110:9300]][indices:data/read/search[phase/fetch/id]]]; nested: ElasticsearchIllegalStateException[No matching token for number_type [BIG_INTEGER]]; "
},
{
"index": "customindex-2015.05.27",
"shard": 2,
"status": 500,
"reason": "RemoteTransportException[[fqdn.omitted.com][inet[/192.168.1.110:9300]][indices:data/read/search[phase/fetch/id]]]; nested: ElasticsearchIllegalStateException[No matching token for number_type [BIG_INTEGER]]; "
},
{
"index": "customindex-2015.05.27",
"shard": 3,
"status": 500,
"reason": "RemoteTransportException[[fqdn.omitted.com][inet[/192.168.112.177:9300]][indices:data/read/search[phase/fetch/id]]]; nested: ElasticsearchIllegalStateException[No matching token for number_type [BIG_INTEGER]]; "
},
{
"index": "customindex-2015.05.27",
"shard": 4,
"status": 500,
"reason": "RemoteTransportException[[fqdn.omitted.com][inet[/192.168.1.110:9300]][indices:data/read/search[phase/fetch/id]]]; nested: ElasticsearchIllegalStateException[No matching token for number_type [BIG_INTEGER]]; "
}
]
},
"hits": {
"total": 86519,
"max_score": null,
"hits": []
},
"aggregations": {
"2": {
"buckets": [
{
"key_as_string": "2015-05-27T05:16:30.000Z",
"key": 1432703790000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:17:00.000Z",
"key": 1432703820000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:17:30.000Z",
"key": 1432703850000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:18:00.000Z",
"key": 1432703880000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:18:30.000Z",
"key": 1432703910000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:19:00.000Z",
"key": 1432703940000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:19:30.000Z",
"key": 1432703970000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:20:00.000Z",
"key": 1432704000000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:20:30.000Z",
"key": 1432704030000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:21:00.000Z",
"key": 1432704060000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:21:30.000Z",
"key": 1432704090000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:22:00.000Z",
"key": 1432704120000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:22:30.000Z",
"key": 1432704150000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:23:00.000Z",
"key": 1432704180000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:23:30.000Z",
"key": 1432704210000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:24:00.000Z",
"key": 1432704240000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:24:30.000Z",
"key": 1432704270000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:25:00.000Z",
"key": 1432704300000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:25:30.000Z",
"key": 1432704330000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:26:00.000Z",
"key": 1432704360000,
"doc_count": 0
},
{
"key_as_string": "2015-05-27T05:26:30.000Z",
"key": 1432704390000,
"doc_count": 4209
},
{
"key_as_string": "2015-05-27T05:27:00.000Z",
"key": 1432704420000,
"doc_count": 7270
},
{
"key_as_string": "2015-05-27T05:27:30.000Z",
"key": 1432704450000,
"doc_count": 6646
},
{
"key_as_string": "2015-05-27T05:28:00.000Z",
"key": 1432704480000,
"doc_count": 7181
},
{
"key_as_string": "2015-05-27T05:28:30.000Z",
"key": 1432704510000,
"doc_count": 6612
},
{
"key_as_string": "2015-05-27T05:29:00.000Z",
"key": 1432704540000,
"doc_count": 6753
},
{
"key_as_string": "2015-05-27T05:29:30.000Z",
"key": 1432704570000,
"doc_count": 6509
},
{
"key_as_string": "2015-05-27T05:30:00.000Z",
"key": 1432704600000,
"doc_count": 10295
},
{
"key_as_string": "2015-05-27T05:30:30.000Z",
"key": 1432704630000,
"doc_count": 13073
},
{
"key_as_string": "2015-05-27T05:31:00.000Z",
"key": 1432704660000,
"doc_count": 14627
},
{
"key_as_string": "2015-05-27T05:31:30.000Z",
"key": 1432704690000,
"doc_count": 3344
}
]
}
}
}
]
}
Template & Mapping
{
"template": "customindex-*",
"settings": {
"index.refresh_integererval": "5s",
"index.number_of_shards": "5"
},
"mappings": {
"_default_": {
"_all": {
"enabled": false
}
},
"netflow": {
"properties": {
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"@version": {
"type": "string"
},
"host": {
"type": "string"
},
"netflow": {
"properties": {
"flow_seq_num": {
"type": "long"
},
"flowset_id": {
"type": "long"
},
"nf_f_conn_id": {
"type": "long"
},
"nf_f_dst_addr_ipv4": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_dst_intf_id": {
"type": "long"
},
"nf_f_dst_port": {
"type": "long"
},
"nf_f_egress_acl_id": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_event_time_msec": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_flow_create_time_msec": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_fwd_flow_delta_bytes": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_rev_flow_delta_bytes": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_flow_bytes": {
"type": "long"
},
"nf_f_fw_event": {
"type": "long"
},
"nf_f_fw_ext_event": {
"type": "long"
},
"nf_f_icmp_code": {
"type": "long"
},
"nf_f_icmp_type": {
"type": "long"
},
"nf_f_icmp_type_ipv6": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_icmp_code_ipv6": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_ingress_acl_id": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_protocol": {
"type": "long"
},
"nf_f_src_addr_ipv4": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_src_intf_id": {
"type": "long"
},
"nf_f_src_port": {
"type": "long"
},
"nf_f_username": {
"type": "string"
},
"nf_f_xlate_dst_addr_ipv4": {
"type": "string"
},
"nf_f_xlate_dst_port": {
"type": "long"
},
"nf_f_xlate_src_addr_ipv4": {
"type": "string",
"index": "not_analyzed"
},
"nf_f_xlate_src_port": {
"type": "long"
},
"version": {
"type": "long"
}
}
},
"type": {
"type": "string"
}
}
}
}
}
Ginja
All 3 comments
This appears to be an elasticsearch issue:
ElasticsearchIllegalStateException[No matching token for number_type [BIG_INTEGER]];
It would appear, as you've indicated, that you've indexed numbers too big for elasticsearch to handle. You'll need to reindex any already indexed data as a string, being aware that you won't be able to aggregate on them. Jump over to http://discuss.elastic.co if you need help
rashidkpc
on 27 May 2015
Done.
What still throws me off is that I'm able to view the data with Kibana 3. And someone mentioned the same thing in this issue thread.
Ginja
on 27 May 2015
i still have the same issue with kibana 5.4
ghost
on 7 Jun 2017
👍15
Was this page helpful?
0 / 5 - 0 ratings
Related issues
nyurik
路
3Comments
stacey-gammon
路
3Comments
LukeMathWalker
路
3Comments
treussart
路
3Comments
celesteking
路
3Comments
Most helpful comment
i still have the same issue with kibana 5.4