-1

我不需要导出 ElasticSearch 中的所有数据并重新索引所有这些数据。

导出Java代码如下。

SearchResponse response = client.prepareSearch("news")
                .setTypes("news_data")
                .setQuery(QueryBuilders.matchAllQuery())
                .setSize(1000)
                .setScroll(new TimeValue(600000))
                .setSearchType(SearchType.SCAN)
                .execute().actionGet();
        String scrollid = response.getScrollId();
        try {
            //把导出的结果以JSON的格式写到文件里
            BufferedWriter out = new BufferedWriter(new FileWriter("es", true));

            while (true) {
                SearchResponse response2 = client.prepareSearchScroll(scrollid)
                        .setScroll(new TimeValue(1000000))
                        .execute().actionGet();
                SearchHits searchHit = response2.getHits();
                //再次查询不到数据时跳出循环
                if (searchHit.getHits().length == 0) {
                    break;
                }
                System.out.println("查询数量 ：" + searchHit.getHits().length);
                for (int i = 0; i < searchHit.getHits().length; i++) {
                    String json = searchHit.getHits()[i].getSourceAsString();
                    out.write(json);
                    out.write("\r\n");
                }
            }
            System.out.println("查询结束");
            out.close();
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

文档数量约为140W。使用这个java代码可以导出60W文件并抛出异常。

远程主机强迫关闭了一个现有的连接
4

1 回答 1

0

对于下一个请求，您必须使用上一个响应中的滚动 ID。

有关更多详细信息，请参阅https://www.elastic.co/guide/en/elasticsearch/reference/1.7/search-request-scroll.html#scroll-scan

也许你可以试试这样的东西？

    SearchResponse response = client.prepareSearch("news")
            .setTypes("news_data")
            .setQuery(QueryBuilders.matchAllQuery())
            .setSize(1000)
            .setScroll(new TimeValue(600000))
            .setSearchType(SearchType.SCAN)
            .execute().actionGet();
    int sequence = 0;
    do
    {
        response = client.prepareSearchScroll(response.getScrollId())
                .setScroll(new TimeValue(600000))
                .execute().actionGet();
        if (response.getHits().getHits().length > 0)
        {
            try
            {
                final BufferedWriter out = new BufferedWriter(new FileWriter("es-" + (++sequence) , true));
                for (final SearchHit hit : response.getHits().getHits())
                {
                    out.write(hit.getSourceAsString());
                    out.write("\r\n");
                }
                out.close();
            }
            catch (final IOException e)
            {
                e.printStackTrace();
            }
        }
    }
    while (response.getHits().hits().length > 0);
于 2016-07-12T12:24:02.673 回答