Ecs: object mapping for [host] tried to parse field [host] as object, but found a concrete value
Hello,
When I try to use this f5ecs template where I integrated the ecs fields I think I will need:
PUT _template/f5ecs
{
"order": 0,
"index_patterns": "f5-002-*",
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": "10000"
}
},
"refresh_interval": "5s",
"number_of_shards": "3",
"number_of_replicas": "1"
}
},
"mappings": {
"doc": {
"_meta": {
"version": "2.0.2"
},
"date_detection": false,
"dynamic": "false",
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "integer"
},
"dig": {
"properties": {
"source": {
"ignore_above": 1024,
"type": "keyword"
},
"app": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"correlation_id": {
"ignore_above": 1024,
"type": "keyword"
},
"message_id": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"destination": {
"properties": {
"domain": {
"ignore_above": 1024,
"type": "keyword"
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"ip": {
"type": "ip"
},
"mac": {
"ignore_above": 1024,
"type": "keyword"
},
"port": {
"type": "long"
},
"subdomain": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"event": {
"properties": {
"action": {
"ignore_above": 1024,
"type": "keyword"
},
"category": {
"ignore_above": 1024,
"type": "keyword"
},
"created": {
"type": "date"
},
"dataset": {
"ignore_above": 1024,
"type": "keyword"
},
"duration": {
"type": "long"
},
"hash": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"raw": {
"doc_values": false,
"ignore_above": 1024,
"index": false,
"type": "keyword"
},
"risk_score": {
"type": "float"
},
"severity": {
"type": "long"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"geoip": {
"properties": {
"city_name": {
"ignore_above": 1024,
"type": "keyword"
},
"continent_name": {
"ignore_above": 1024,
"type": "keyword"
},
"country_iso_code": {
"ignore_above": 1024,
"type": "keyword"
},
"location": {
"type": "geo_point"
},
"region_name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"host": {
"properties": {
"architecture": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"ip": {
"type": "ip"
},
"mac": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"properties": {
"family": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"platform": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"timezone": {
"properties": {
"offset": {
"properties": {
"sec": {
"type": "long"
}
}
}
}
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"http": {
"properties": {
"response": {
"properties": {
"body": {
"norms": false,
"type": "text"
},
"status_code": {
"type": "long"
}
}
}
}
},
"log": {
"properties": {
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"line": {
"type": "long"
},
"message": {
"doc_values": false,
"ignore_above": 1024,
"index": false,
"type": "keyword"
},
"offset": {
"type": "long"
}
}
},
"message": {
"norms": false,
"type": "text"
},
"network": {
"properties": {
"direction": {
"ignore_above": 1024,
"type": "keyword"
},
"forwarded_ip": {
"type": "ip"
},
"inbound": {
"properties": {
"bytes": {
"type": "long"
},
"packets": {
"type": "long"
}
}
},
"outbound": {
"properties": {
"bytes": {
"type": "long"
},
"packets": {
"type": "long"
}
}
},
"protocol": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"organization": {
"properties": {
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"norms": false,
"type": "text"
}
}
},
"os": {
"properties": {
"family": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"platform": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"process": {
"properties": {
"args": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"pid": {
"type": "long"
},
"ppid": {
"type": "long"
},
"title": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"service": {
"properties": {
"ephemeral_id": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"state": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"source": {
"properties": {
"domain": {
"ignore_above": 1024,
"type": "keyword"
},
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"ip": {
"type": "ip"
},
"mac": {
"ignore_above": 1024,
"type": "keyword"
},
"port": {
"type": "long"
},
"subdomain": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"tls": {
"properties": {
"certificates": {
"doc_values": false,
"type": "keyword"
},
"ciphersuite": {
"ignore_above": 1024,
"type": "keyword"
},
"servername": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"url": {
"properties": {
"fragment": {
"ignore_above": 1024,
"type": "keyword"
},
"host": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"href": {
"fields": {
"raw": {
"ignore_above": 1024,
"type": "keyword"
}
},
"norms": false,
"type": "text"
},
"password": {
"ignore_above": 1024,
"type": "keyword"
},
"path": {
"fields": {
"raw": {
"ignore_above": 1024,
"type": "keyword"
}
},
"norms": false,
"type": "text"
},
"port": {
"type": "long"
},
"query": {
"fields": {
"raw": {
"ignore_above": 1024,
"type": "keyword"
}
},
"norms": false,
"type": "text"
},
"scheme": {
"ignore_above": 1024,
"type": "keyword"
},
"username": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"user": {
"properties": {
"email": {
"ignore_above": 1024,
"type": "keyword"
},
"hash": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"user_agent": {
"properties": {
"device": {
"ignore_above": 1024,
"type": "keyword"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"properties": {
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"patch": {
"ignore_above": 1024,
"type": "keyword"
},
"raw": {
"norms": false,
"type": "text"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"f5": {
"properties": {
"apd": {
"properties": {
"function": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"session": {
"properties": {
"key": {
"ignore_above": 1024,
"type": "keyword"
},
"value": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"processor": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"line_number": {
"type": "long"
},
"message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
}
}
},
"dcc": {
"properties": {
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
},
"transaction": {
"ignore_above": 1024,
"type": "keyword"
},
"drop_counter": {
"type": "long"
},
"evasion_violation": {
"ignore_above": 1024,
"type": "keyword"
},
"event": {
"ignore_above": 1024,
"type": "keyword"
},
"http_violation": {
"ignore_above": 1024,
"type": "keyword"
},
"http_classifier": {
"ignore_above": 1024,
"type": "keyword"
},
"injection_ratio": {
"ignore_above": 1024,
"type": "keyword"
},
"injection_threshold": {
"ignore_above": 1024,
"type": "keyword"
},
"legit_sessions": {
"type": "long"
},
"new_transactions": {
"ignore_above": 1024,
"type": "keyword"
},
"operation_mode": {
"ignore_above": 1024,
"type": "keyword"
},
"request": {
"ignore_above": 1024,
"type": "keyword"
},
"rest": {
"ignore_above": 1024,
"type": "keyword"
},
"route_domain": {
"ignore_above": 1024,
"type": "keyword"
},
"scheme": {
"ignore_above": 1024,
"type": "keyword"
},
"scraping_status": {
"ignore_above": 1024,
"type": "keyword"
},
"scraping_type": {
"ignore_above": 1024,
"type": "keyword"
},
"session_id": {
"ignore_above": 1024,
"type": "keyword"
},
"support_id": {
"ignore_above": 1024,
"type": "keyword"
},
"violation": {
"ignore_above": 1024,
"type": "keyword"
},
"violation_counter": {
"type": "long"
},
"virus_name": {
"ignore_above": 1024,
"type": "keyword"
},
"web_violation": {
"ignore_above": 1024,
"type": "keyword"
},
"xff_ip": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"correlation_id": {
"ignore_above": 1024,
"type": "keyword"
},
"message_id": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"f5_httpd_message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"f5_httpd_user_name": {
"ignore_above": 256,
"type": "keyword"
},
"f5_message_id": {
"ignore_above": 16,
"type": "keyword"
},
"f5_session_id": {
"ignore_above": 16,
"type": "keyword"
},
"f5_ssh_message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"f5_ssh_port": {
"type": "keyword"
},
"f5_ssh_source_ip": {
"type": "ip"
},
"f5_ssh_sourceip": {
"type": "ip"
},
"f5_ssh_source_port": {
"type": "keyword"
},
"f5_ssh_username": {
"type": "keyword"
},
"f5_tmm_auth_id": {
"type": "keyword"
},
"f5_tmm_auth_ip": {
"type": "ip"
},
"f5_tmm_auth_message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"f5_tmm_auth_port": {
"type": "keyword"
},
"f5_tmm_auth_type": {
"type": "keyword"
},
"f5_tmm_auth_version": {
"type": "keyword"
},
"f5_tmm_client_activex": {
"type": "integer"
},
"f5_tmm_client_browser": {
"type": "keyword"
},
"f5_tmm_client_browser_version": {
"type": "keyword"
},
"f5_tmm_client_cpu": {
"type": "keyword"
},
"f5_tmm_client_ip": {
"type": "ip"
},
"f5_tmm_client_javascript": {
"type": "integer"
},
"f5_tmm_client_platform": {
"type": "keyword"
},
"f5_tmm_client_plugin": {
"type": "integer"
},
"f5_tmm_client_port": {
"type": "keyword"
},
"f5_tmm_client_ui_mode": {
"type": "keyword"
},
"f5_tmm_event": {
"type": "keyword"
},
"f5_tmm_message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"f5_tmm_reputation": {
"type": "keyword"
},
"f5_tmm_rest": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"f5_tmm_rule": {
"type": "keyword"
},
"f5_tmm_rule_message": {
"type": "text",
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
}
},
"f5_tmm_sequence_id": {
"type": "keyword"
},
"f5_tmm_server_ip": {
"type": "ip"
},
"f5_tmm_server_port": {
"type": "integer"
},
"f5_tmm_session_bytes_in": {
"type": "long"
},
"f5_tmm_session_bytes_out": {
"type": "long"
},
"f5_tmm_session_client_ip": {
"type": "ip"
},
"f5_tmm_session_deleted_reason": {
"type": "keyword"
},
"f5_tmm_session_listener": {
"type": "keyword"
},
"f5_tmm_session_location": {
"type": "keyword"
},
"f5_tmm_session_vip_ip": {
"type": "ip"
},
"f5_tmm_type": {
"type": "keyword"
}
}
}
},
"aliases": {
"f5": {}
}
}
I get Logstash errors like:
[2018-07-06T15:46:38,453][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>["index", {:_id=>nil, :_index=>"f5-002-2018.07.06", :_type=>"doc", :_routing=>nil}, #<LogStash::Event:0x2373a721>], :response=>{"index"=>{"_index"=>"f5-002-2018.07.06", "_type"=>"doc", "_id"=>"hGDYb2QBpfUnuaeQN_7m", "status"=>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"object mapping for [host] tried to parse field [host] as object, but found a concrete value"}}}}
GET /_cat/templates/f5*?v&s=name:asc
name index_patterns order version
f5 [f5-001-*] 0
f5ecs [f5-002-*] 0
And my pipeline:
input {
udp {
type => 'syslog-f5'
port => 5548
id => 'input-syslog-f5'
}
}
filter {
grok {
patterns_dir => "/etc/logstash/patterns"
match => [ "message", "\A<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} (slot1\/)?%{HOSTNAMEUND:host.name} %{LOGLEVEL:event.severity} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}\Z" ]
add_tag => "grok_f5"
id => 'grok-syslog-f5'
}
translate {
dictionary_path => [ "/etc/logstash/dictionaries/syslogpri.yml" ]
field => "syslog_pri"
destination => "log.level"
id => 'translate-log-level'
}
}
output {
elasticsearch {
index => "f5-002-%{+YYYY.MM.dd}"
hosts => ["https://srvlogstashqa01.gentgrp.gent.be:9200"]
manage_template => false
user => "logstash_internal"
password => "${LOGSTASH_INTERNAL}"
document_type => doc
}
}
the f5 template for f5-001-* still has a 'host' field, but shouldn't interfer as the my new f5ecs template applies to a different index?. I'm not sure what's going wrong here, will have to investigate further, but I thought I throw it in here, it might be related to the way I refer to the host object in my pipeline?
willemdh
All 13 comments
Hmmm, the UDP input sets a host field. And it doesn't look like it's easy to configure this away (const, field being set).
Your best bet for now is probably to add a mutate to delete or rename that field in your LS pipeline.
I'll open an issue on UDP input to move to a more ECS friendly location for the hostname.
webmat
on 6 Jul 2018
馃檪 Glad to hear it's not a config error on my side.. I'll test your suggestion asap.
willemdh
on 6 Jul 2018
Coincidentally, I opened a PR yesterday to fix this: https://github.com/logstash-plugins/logstash-input-udp/pull/41
praseodym
on 6 Jul 2018
Filed issue https://github.com/logstash-plugins/logstash-input-udp/issues/42 for this.
webmat
on 6 Jul 2018
Adding
mutate {
remove_field => [ "host" ]
}
solved my issue. Thanks. I'll go ahead and close this. Tx @webmat @praseodym
willemdh
on 6 Jul 2018
Hi,
I keep getting the similar error.
object mapping for [requestnode] tried to parse field [null] as object, but found a concrete value
But a simple renaming of the field from 'requestnode' to 'request_node' solves the problem. Why is that? Is requestnode a special keyword for elastic? I've had the same error for multiple such fields before and every time just renaming it solved the issue.
But weird thing is that the same field name has been working all along. And now suddenly elasticsearch throws out error for a new document. Why is that? And why does putting a _ in the field name makes the error vanish?
sarangdharmapurikar1
on 8 Mar 2019
I apologize for the inconvenience, but this is an Elasticsearch usage question, and should be
asked at https://discuss.elastic.co.
GitHub should be used for reporting bugs, suggesting improvement and opening pull requests.
webmat
on 8 Mar 2019
For those who are facing the same issue, Here is how we solved it
processors:
- decode_json_fields:
fields: ["message"]
process_array: false
max_depth: 3
target: ""
overwrite_keys: true
- rename:
fields:
- from: "host"
to: "host.name"
Interesting observation is the location of rename processing. It should be after dropping, decoding your input logs.
Hope it will resolve your issue without updating logstash configuration.
nitinpatil1992
on 25 Nov 2019
For those who are facing the same issue, Here is how we solved it
processors: - decode_json_fields: fields: ["message"] process_array: false max_depth: 3 target: "" overwrite_keys: true - rename: fields: - from: "host" to: "host.name"Interesting observation is the location of
renameprocessing. It should be after dropping, decoding your input logs.
Hope it will resolve your issue without updating logstash configuration.
I don't understand where this yaml-like markup you wrote is applied. It cannot be applied in Kibana Dev Tools for ElasticSearch because that uses JSON format.
willquill
on 27 Dec 2019
I don't understand where this yaml-like markup you wrote is applied. It cannot be applied in Kibana Dev Tools for ElasticSearch because that uses JSON format.
That is a Filebeat config (using the rename processor).
praseodym
on 28 Dec 2019
Adding
mutate { remove_field => [ "host" ] }solved my issue. Thanks. I'll go ahead and close this. Tx @webmat @praseodym
Is this because logstash sends data to elastic and the ip is ipv6? After adding this filter, there is no ip, logstash cannot parse ipv6, I guess
haoqi123
on 21 Mar 2021
@haoqi123 No, when this issue was originally opened the logstash-input-udp plugin was setting host field to a value instead of an object. This then clashed with the host.* fieldset object defined in ECS and specified in this provided template.
The mutate config was a workaround to drop the host field added by the UDP input plugin before Logstash tried outputting to Elasticsearch, but a fix has been addressed in that plugin for some time now: https://github.com/logstash-plugins/logstash-input-udp/pull/43.
If you have any other questions or issues, feel free to open another issue or a topic thread over on discuss.elastic.co.
ebeahan
on 22 Mar 2021
澧炲姞杩欎釜灏辨病鐢ㄦ姤閿欎簡
mutate {
remove_field => [ "host" ]
}
twz999
on 12 May 2021
Related issues
willemdh
路
5Comments
Forbo
路
4Comments
faudebert
路
3Comments
rw-access
路
5Comments
mikeh-elastic
路
6Comments
Most helpful comment
Adding
solved my issue. Thanks. I'll go ahead and close this. Tx @webmat @praseodym