1

我编写了一个 Logstash 配置文件，其中包含三个文件输入（它们都是日志文件）、三个过滤器（每个过滤器具有不同的模式）和三个弹性搜索输出（每个都进入不同的索引）。每个索引根据其输入类型具有不同的模板，并且索引按周分区。

该问题发生在所描述的配置文件运行时，索引模板被忽略并且不会对索引创建生效。

模板在这种情况下不起作用：

input {
    file {
        path => ["/path/to/file.log"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        type => 'type_1'
        }
    file {
        path => "/path/to/file2.log"
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        type => 'type_2'
    }
    file {
        path => ["/path/to/file3.log"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        type => 'type_3'
    }
}
filter {
    if [type] == "type_1" {
        csv {
            columns => ["column1","column2","column3"]
                separator => "|"
        }
        date {
            match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "@timestamp"
        }
        date {
            match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "column3"
        }   
        mutate {
            remove_field => [ "message" ]
            remove_field => [ "path" ]
            remove_field => [ "host" ]
        }
    }
    if [type] == "type_2" {
        csv {
            columns => ["column1","column2",]
            separator => "|"
        }
        mutate {
            remove_field => [ "message" ]
            remove_field => [ "path" ]
            remove_field => [ "host" ]
            convert => { "column1" => "float" }
        }
        date {
            match => [ "column2", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "@timestamp"
        }
        date {
            match => [ "column2", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "column2"
        }
    }
    if [type] == "type_3" {
        csv {
            columns => ["column1","column2","column3","column4"]
            separator => "|"
        }
        mutate {
            remove_field => [ "message" ]
            remove_field => [ "path" ]
            remove_field => [ "host" ]
            convert => { "column3" => "float" }
            convert => { "column1" => "float" }
        }
        date {
            match => [ "column4", "EEE MMM dd HH:mm:ss zzz yyyy" ]
            target => "@timestamp"
        }
        date {
             match => [ "column4", "EEE MMM dd HH:mm:ss zzz yyyy" ]
             target => "column4"
        }
    }
}
output {
    if [type] == "type_1" {
        elasticsearch { 
            hosts => ["localhost:9200"]
            index => "type_1-%{+xxxx.ww}"
        }
    }
    if [type] == "type_2" {
        elasticsearch { 
            hosts => ["localhost:9200"]
            index => "type_2-%{+xxxx.ww}"
        }
    }
    if [type] == "type_3" {
        elasticsearch {
            hosts => ["localhost:9200"]
            index => "type_3-%{+xxxx.ww}"
        }
    }
}

相反，当每个输入文件类型、过滤器和弹性搜索输出使用单个配置文件时，模板可以正常工作。

模板在这里工作正常：

input {
    file {
        path => ["/path/to/file.log"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        ignore_older => 0
        }
}
filter {
    csv {
        columns => ["column1","column2","column3"]
            separator => "|"
    }
    date {
        match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
        target => "@timestamp"
    }
    date {
        match => [ "column3", "EEE MMM dd HH:mm:ss zzz yyyy" ]
        target => "column3"
    }   
    mutate {
        remove_field => [ "message" ]
        remove_field => [ "path" ]
        remove_field => [ "host" ]
    }
}
output {
    elasticsearch { 
        hosts => ["localhost:9200"]
        index => "type_1-%{+xxxx.ww}"
    }
}

我已经在配置文件中使用了以下参数：

  • 模板 => “file_name.json”
  • 模板覆盖=>“真”
  • 管理模板 => “真”
  • 模板名称 => "模板名称"

但他们没有帮助。

以前有人遇到过这个错误吗？

（我正在使用 elasticsearch 2.3.2 和 logstash 2.3.2）

任何帮助将不胜感激

我的模板

type_1

 curl -X PUT 'localhost:9200/_template/type_1' -d '
    {
      "template": "type_1*", 
      "settings" : {
        "index" : {
          "refresh_interval" : "30s"
        }
      },
      "mappings": {
        "logs" : {
            "_all": {
              "enabled": false
            },
            "_source": {
              "enabled": false
            },
            "dynamic": "strict",
            "properties" : {
                 "column3" : {
                    "type" : "date",
                    "format" : "strict_date_optional_time||epoch_millis",
                        "norms": {
                            "enabled": false
                        }
                  },
                  "@timestamp" : {
                    "format" : "strict_date_optional_time||epoch_millis",
                    "type" : "date",
                                "norms": {
                                    "enabled": false
                                }
                  },
                  "column2" : {
                    "type" : "string",
                    "index": "not_analyzed"             
                  },
                  "column1" : {
                    "type" : "string",
                    "index": "not_analyzed"
                  },
                  "@version" : {
                    "type" : "string",
                        "norms": {
                            "enabled": false
                        }
                  }
             }
         }
      }
    }';

类型_2

 curl -X PUT 'localhost:9200/_template/type_2' -d '
    {
      "template": "type_2*", 
      "settings" : {
        "index" : {
          "refresh_interval" : "30s"
        }
      },
      "mappings": {
        "logs" : {
            "_all": {
              "enabled": false
            },
            "_source": {
              "enabled": false
            },
            "dynamic": "strict",
            "properties" : {
                 "column2" : {
                    "type" : "date",
                    "format" : "strict_date_optional_time||epoch_millis",
                        "norms": {
                            "enabled": false
                        }
                  },
                  "@timestamp" : {
                    "format" : "strict_date_optional_time||epoch_millis",
                    "type" : "date",
                                "norms": {
                                    "enabled": false
                                }
                  },
                  "column1" : {
                    "type" : "float",
                    "index": "not_analyzed"             
                  },
                  "@version" : {
                    "type" : "string",
                        "norms": {
                            "enabled": false
                        }
                  }
             }
         }
      }
    }';

类型_3

curl -X PUT 'localhost:9200/_template/type_3' -d '
{
  "template": "type_3*", 
  "settings" : {
    "index" : {
      "refresh_interval" : "30s"
    }
  },
  "mappings": {
    "logs" : {
        "_all": {
          "enabled": false
        },
        "_source": {
          "enabled": false
        },
        "dynamic": "strict",
        "properties" : {
             "column4" : {
                "type" : "date",
                "format" : "strict_date_optional_time||epoch_millis",
                    "norms": {
                        "enabled": false
                    }
              },
              "@timestamp" : {
                "format" : "strict_date_optional_time||epoch_millis",
                "type" : "date",
                            "norms": {
                                "enabled": false
                            }
              },
              "column3" : {
                "type" : "float",
                "index": "not_analyzed"             
              },
              "column2" : {
                "type" : "string",
                "index": "not_analyzed"             
              },
               "column1" : {
                "type" : "float",
                "index": "not_analyzed"             
              },
              "@version" : {
                "type" : "string",
                    "norms": {
                        "enabled": false
                    }
              }
         }
     }
  }
}';
4

1 回答 1

0

我找到了解决方案

Losgstash 配置文件使用 type => 'type_1' 来区分输入文件、过滤器和 Elasticsearch 输出，还定义了一个模板类型和一个名为“type”的新字段。

我们在映射模板“logs”中使用了默认类型，并且我们没有对 logstash 配置文件中定义的 'type_1' 进行 cosdifering。由于这个原因，模板被忽略了。

解决方案：更改模板映射中的类型问题已解决。

例如：

curl -X PUT 'localhost:9200/_template/type_1' -d '
    {
      "template": "type_1*", 
      "settings" : {
        "index" : {
          "refresh_interval" : "30s"
        }
      },
      "mappings": {
        "type_1" : {
            "_all": {
              "enabled": false
            },
            "_source": {
              "enabled": false
            },
            "dynamic": "strict",
            "properties" : {
                 "column3" : {
                    "type" : "date",
                    "format" : "strict_date_optional_time||epoch_millis",
                        "norms": {
                            "enabled": false
                        }
                  },
                  "@timestamp" : {
                    "format" : "strict_date_optional_time||epoch_millis",
                    "type" : "date",
                                "norms": {
                                    "enabled": false
                                }
                  },
                  "column2" : {
                    "type" : "string",
                    "index": "not_analyzed"             
                  },
                  "column1" : {
                    "type" : "string",
                    "index": "not_analyzed"
                  },
                  "@version" : {
                    "type" : "string",
                        "norms": {
                            "enabled": false
                        }
                  }
             }
         }
      }
    }';
于 2016-09-26T18:23:36.643 回答