Kibana: [ML] CSV data import ignoring mappings on cloud

Created on 29 Aug 2019 · 3Comments · Source: elastic/kibana

I'm on 7.3.0 on Cloud and am running into issues uploading a CSV file to my could instance as well. The mappings seem to be ignored but the ingest pipeline is working. All rows seem to get in, but the mappings are completely ignored. All I'm trying to do, and can successfully do locally, is build a geo_point from the two columns lat and long. The new location field is ignoring the geo_point mappings and converting to a string.

Data set below
get_it_done_2019_requests_datasd.csv.zip

Mappings I put in the file uploader

{
  "@timestamp": {
    "type": "date"
  },
  "case_age_days": {
    "type": "long"
  },
  "case_origin": {
    "type": "keyword"
  },
  "case_record_type": {
    "type": "keyword"
  },
  "comm_plan_code": {
    "type": "long"
  },
  "comm_plan_name": {
    "type": "keyword"
  },
  "council_district": {
    "type": "keyword"
  },
  "lat": {
    "type": "double"
  },
  "long": {
    "type": "double"
  },
  "park_name": {
    "type": "keyword"
  },
  "public_description": {
    "type": "text"
  },
  "referred": {
    "type": "text"
  },
  "requested_datetime": {
    "type": "date",
    "format": "iso8601"
  },
  "sap_notification_number": {
    "type": "long"
  },
  "service_name": {
    "type": "keyword"
  },
  "service_request_id": {
    "type": "long"
  },
  "service_request_parent_id": {
    "type": "long"
  },
  "status": {
    "type": "keyword"
  },
  "location": {
    "type": "geo_point"
  }
}

Ingest pipeline

{
  "processors": [
    {
      "set": {
        "field": "location",
        "value": "{{lat}},{{long}}"
      }
    }
  ]
}

Mappings post-ingest

{
  "mapping": {
    "_meta": {
      "created_by": "ml-file-data-visualizer"
    },
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "case_age_days": {
        "type": "long"
      },
      "case_origin": {
        "type": "keyword"
      },
      "case_record_type": {
        "type": "keyword"
      },
      "comm_plan_code": {
        "type": "long"
      },
      "comm_plan_name": {
        "type": "keyword"
      },
      "council_district": {
        "type": "long"
      },
      "lat": {
        "type": "double"
      },
      "location": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "long": {
        "type": "double"
      },
      "park_name": {
        "type": "keyword"
      },
      "public_description": {
        "type": "text"
      },
      "referred": {
        "type": "text"
      },
      "requested_datetime": {
        "type": "date",
        "format": "iso8601"
      },
      "sap_notification_number": {
        "type": "long"
      },
      "service_name": {
        "type": "keyword"
      },
      "service_request_id": {
        "type": "long"
      },
      "service_request_parent_id": {
        "type": "long"
      },
      "status": {
        "type": "keyword"
      },
      "updated_datetime": {
        "type": "date",
        "format": "iso8601"
      }
    }
  }
}

:ml bug

Source