fluentd has very high cpu usage

Created on 21 Jun 2018 · 3Comments · Source: fluent/fluentd

Check CONTRIBUTING guideline first and here is the list to help us investigate the problem.

fluentd
RHEL7.5
We consume Collectd through the write_http plugin and send them to elasticsearch
config file:

# HTTP input
<source>
  @type http
  @id http_input
  @log_level debug

  port 9880
  bind localhost
</source>

<filter ovirt_collectd>
  @type record_transformer
  <record>
    timestamp ${time}
  </record>
</filter>

<match ovirt_collectd>
  @type rewrite_tag_filter
  rewriterule1 plugin ^(\w+\S+) ${tag}.$1
</match>

<match ovirt_collectd.**>
  @type collectd_nest
  add_tag_prefix nested
</match>

<filter nested.ovirt_collectd.**>
  @type record_transformer
  enable_ruby
  auto_typecast true
  <record>
    tag project.ovirt-metrics-engine
    hostname #{Socket.gethostname}
    ovirt {"entity":"engine","engine_fqdn":"rhevm.eng.lab.tlv.redhat.com"}
    ipaddr4 10.35.16.209
    ovirt_index_name project.ovirt-metrics-engine..${Time.parse("${record['collectd']['timestamp']}").strftime('%Y.%m.%d')}
    # timestamp example: "timestamp":"2017-05-14 09:44:59 +0300" to iso8601 format
    time ${DateTime.strptime("${record['collectd']['timestamp']}", "%Y-%m-%d %H:%M:%S %z").iso8601}
    for_remove ${record["collectd"].delete("timestamp")}
  </record>
    remove_keys for_remove
</filter>

<filter **>
  @type viaq_data_model
  rename_time_if_missing "true"
</filter>

<match fluent.**>
  @type stdout
</match>

<match nested.ovirt_collectd**>
  @type elasticsearch
  host metrics-store.eng.lab.
  port 9200
  scheme https
  client_cert /etc/fluentd/elasticsearch_client_cert.pem
  client_key /etc/fluentd/elasticsearch_client_key.pem
  ca_file /etc/fluentd/elasticsearch_ca_cert.pem
  ssl_verify false
  target_index_key ovirt_index_name
  remove_keys ovirt_index_name
  type_name com.redhat.viaq.common
  request_timeout 600

# Buffer configurations
  flush_interval 10s
  buffer_chunk_limit 32m
  buffer_queue_limit 8
  buffer_queue_full_action drop_oldest_chunk
  retry_wait 0s
  retry_limit 0
  disable_retry_limit False
  max_retry_wait 0
  flush_at_shutdown False
  num_threads 5
  slow_flush_log_threshold 20.0
</match>