3

如何更改映射以解决这些错误,弹性堆栈的新手

` [logstash.outputs.elasticsearch] 无法将事件索引到 Elasticsearch。{:status=>400, :action=>["index", {:_id=>nil, :_index=>"cloudtrail-2018.10.08", :_type=>"doc", :_routing=>nil}, #], :response=>{"index"=>{"_index"=>"cloudtrail-2018.10.08", "_type"=>"doc", "_id"=>"CkkQVWYBeGi09oGfr-kl", "status" =>400, "error"=>{"type"=>"mapper_parsing_exception", "reason"=>"[requestParameters.filter] 的对象映射试图将字段 [null] 解析为对象,但找到了具体值"} }}}

`这是我的aws es上的映射,我基本上是在尝试使用logstash将s3中的cloudtrail日志摄取到elasticsearch和kibana,但目前似乎没有工作。需要更多帮助并了解为什么不是所有数据都显示在 kibana 中

{
  "cloudtrail-2018.09.26": {
    "mappings": {
      "_default_": {
        "dynamic_templates": [
          {
            "message_field": {
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "norms": false,
                "type": "text"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "norms": false,
                "type": "text"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "keyword"
          },
          "geoip": {
            "dynamic": "true",
            "properties": {
              "ip": {
                "type": "ip"
              },
              "latitude": {
                "type": "half_float"
              },
              "location": {
                "type": "geo_point"
              },
              "longitude": {
                "type": "half_float"
              }
            }
          }
        }
      },
      "doc": {
        "dynamic_templates": [
          {
            "message_field": {
              "path_match": "message",
              "match_mapping_type": "string",
              "mapping": {
                "norms": false,
                "type": "text"
              }
            }
          },
          {
            "string_fields": {
              "match": "*",
              "match_mapping_type": "string",
              "mapping": {
                "fields": {
                  "keyword": {
                    "ignore_above": 256,
                    "type": "keyword"
                  }
                },
                "norms": false,
                "type": "text"
              }
            }
          }
        ],
        "properties": {
          "@timestamp": {
            "type": "date"
          },
          "@version": {
            "type": "keyword"
          },
          "additionalEventData": {
            "properties": {
              "LoginTo": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "MFAUsed": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "MfaType": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "MobileVersion": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "SamlProviderArn": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "configRuleArn": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "configRuleInputParameters": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "configRuleName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "managedRuleIdentifier": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "notificationJobType": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "service": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "vpcEndpointId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }
            }
          },
          "apiVersion": {
            "type": "date"
          },
          "awsRegion": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "errorCode": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "errorMessage": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventID": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventName": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventSource": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventType": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "eventVersion": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "geoip": {
            "dynamic": "true",
            "properties": {
              "ip": {
                "type": "ip"
              },
              "latitude": {
                "type": "half_float"
              },
              "location": {
                "type": "geo_point"
              },
              "longitude": {
                "type": "half_float"
              }
            }
          },
          "managementEvent": {
            "type": "boolean"
          },
          "readOnly": {
            "type": "boolean"
          },
          "recipientAccountId": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "requestID": {
            "type": "text",
            "norms": false,
            "fields": {
              "keyword": {
                "type": "keyword",
                "ignore_above": 256
              }
            }
          },
          "requestParameters": {
            "properties": {
              "DescribeHostsRequest": {
                "properties": {
                  "Filter": {
                    "properties": {
                      "Name": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "Value": {
                        "properties": {
                          "content": {
                            "type": "text",
                            "norms": false,
                            "fields": {
                              "keyword": {
                                "type": "keyword",
                                "ignore_above": 256
                              }
                            }
                          },
                          "tag": {
                            "type": "long"
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  },
                  "MaxResults": {
                    "type": "long"
                  }
                }
              },
              "DescribeInstanceCreditSpecificationsRequest": {
                "properties": {
                  "InstanceId": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeLaunchTemplatesRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  }
                }
              },
              "DescribeNatGatewaysRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  },
                  "NatGatewayId": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeSecurityGroupReferencesRequest": {
                "properties": {
                  "GroupId": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeStaleSecurityGroupsRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  },
                  "VpcId": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  }
                }
              },
              "DescribeVolumesModificationsRequest": {
                "properties": {
                  "MaxResults": {
                    "type": "long"
                  }
                }
              },
              "DescribeVpcClassicLinkDnsSupportRequest": {
                "properties": {
                  "VpcIds": {
                    "properties": {
                      "content": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      },
                      "tag": {
                        "type": "long"
                      }
                    }
                  }
                }
              },
              "DescribeVpcEndpointsRequest": {
                "type": "object"
              },
              "aRN": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "accountAttributeNameSet": {
                "properties": {
                  "items": {
                    "properties": {
                      "attributeName": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      }
                    }
                  }
                }
              },
              "accountId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "acl": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "actionPrefix": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "agentName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "agentStatus": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "agentVersion": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "allocationId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "allocationIdsSet": {
                "properties": {
                  "items": {
                    "properties": {
                      "allocationId": {
                        "type": "text",
                        "norms": false,
                        "fields": {
                          "keyword": {
                            "type": "keyword",
                            "ignore_above": 256
                          }
                        }
                      }
                    }
                  }
                }
              },
              "allowUnauthenticatedIdentities": {
                "type": "boolean"
              },
              "applicationName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "associationFilterList": {
                "properties": {
                  "key": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  },
                  "value": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  }
                }
              },
              "associationId": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "assumeRolePolicyDocument": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "attachment": {
                "properties": {
                  "attachmentId": {
                    "type": "text",
                    "norms": false,
                    "fields": {
                      "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                      }
                    }
                  },
                  "deleteOnTermination": {
                    "type": "boolean"
                  }
                }
              },
              "attribute": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "attributeNames": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "attributeType": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "autoCreateApplication": {
                "type": "boolean"
              },
              "autoScalingGroupName": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "autoScalingGroupNames": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              },
              "availabilityZone": {
                "type": "text",
                "norms": false,
                "fields": {
                  "keyword": {
                    "type": "keyword",
                    "ignore_above": 256
                  }
                }
              }

提前致谢。

4

1 回答 1

3

抱歉,有点晚了。我遇到了同样的问题,这就是我发现的:自 ES 6.4 以来,您无法更改映射。您必须删除现有索引并重新创建它。这就是我所做的,而且效果很好

https://www.elastic.co/guide/en/elasticsearch/reference/6.4/indices-delete-mapping.html

于 2019-07-19T19:55:19.913 回答