ElasticSearch 查询所有数据_elasticsearch查询数据条数

1，分页查询所有，但最多能查10000 条记录，超过就会报错

public List getAll(String index, String type, QueryBuilder queryBuilder, Class tClass) {

SearchResponse searchResponse = buildRequest(index, type, queryBuilder, 0).get();

SearchHits hits = searchResponse.getHits();

int total = (int) hits.totalHits;

List result = new ArrayList<>(total);

addToResult(hits,result,tClass);

int pagesize = 1000;

int page = total / pagesize;

if(total % pagesize != 0){

page++;

}

for (int i = 2; i <= page; i++) {

int from = (i - 1) * pagesize;

searchResponse = buildRequest(index, type, queryBuilder, from).get();

hits = searchResponse.getHits();

addToResult(hits,result,tClass);

}

return result;

}

private SearchRequestBuilder buildRequest(String index,String type,QueryBuilder queryBuilder,int from) {

TransportClient client = elasticSearchManager.getClient(“ad”);

int pagesize = 1000;

SearchRequestBuilder builder = client

.prepareSearch(index)

.setTypes(type)

.setFrom(from)

.setSize(pagesize)

.setQuery(queryBuilder);

log.info("ElasticsearchDAO query string： GET {}/{}/_search {} ", index, type, builder.toString());

return builder;

}

2,利用scroll，这是官方推荐的方式

import lombok.extern.slf4j.Slf4j;

import org.elasticsearch.action.search.SearchRequestBuilder;

import org.elasticsearch.action.search.SearchResponse;

import org.elasticsearch.client.transport.TransportClient;

import org.elasticsearch.common.unit.TimeValue;

import org.elasticsearch.index.query.BoolQueryBuilder;

import org.elasticsearch.index.query.QueryBuilder;

import org.elasticsearch.index.query.QueryBuilders;

import org.elasticsearch.search.SearchHit;

import org.elasticsearch.search.SearchHits;

import org.elasticsearch.search.aggregations.Aggregation;

import org.elasticsearch.search.aggregations.AggregationBuilder;

import org.elasticsearch.search.aggregations.Aggregations;

import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;

import org.elasticsearch.search.aggregations.bucket.terms.Terms;

import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.stereotype.Component;

import org.springframework.util.CollectionUtils;

import java.io.IOException;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.concurrent.ExecutionException;

@Slf4j

@Component

public class ElasticsearchDAO {

@Autowired

private ElasticSearchManager elasticSearchManager;

private int pagesize = 10000;

/**

* es 游标有效期

*/

private final int ES_SCROLL_TIME = 10;

public List search(SearchRequest req) throws ExecutionException, InterruptedException {

BoolQueryBuilder boolQuery = req.getQueryBuilder();

List result = getAll(Config.ES_REPORT_INDEX, Config.ES_REPORT_TYPE, boolQuery,ReportDO.class);

return result;

}

/**

* 查询所有

*/

private List getAll(String index, String type, QueryBuilder queryBuilder, Class tClass) throws ExecutionException, InterruptedException {

SearchResponse searchResponse = buildRequest(index, type, queryBuilder).get();

String scrollId = searchResponse.getScrollId();

SearchHits hits = searchResponse.getHits();

List result = new ArrayList<>();

addToResult(hits,result,tClass);

while (true){

List list = getScrollResult(scrollId, tClass, result);

if(list == null){

break;

}

}

return result;

}

private List getScrollResult(String scrollId,Class tClass, List result) throws ExecutionException, InterruptedException {

TransportClient client = elasticSearchManager.getClient(“ad”);

SearchResponse searchResponse = client.prepareSearchScroll(scrollId) //设置游标

.setScroll(TimeValue.timeValueSeconds(ES_SCROLL_TIME)) //设置游标有效期

.execute()

.get();

SearchHits hits = searchResponse.getHits();

SearchHit[] searchHits = hits.getHits();

if(searchHits == null || searchHits.length == 0){

return null;

}

addToResult(hits,result,tClass);

return result;

}

private SearchRequestBuilder buildRequest(String index,String type,QueryBuilder queryBuilder) {

TransportClient client = elasticSearchManager.getClient(“ad”);

SearchRequestBuilder builder = client

.prepareSearch(index)

.setTypes(type)

.setScroll(TimeValue.timeValueSeconds(ES_SCROLL_TIME)) //设置游标有效期

.setSize(pagesize)

.setQuery(queryBuilder);

log.info("ElasticsearchDAO query string： GET {}/{}/_search {} ", index, type, builder.toString());

return builder;

}

private void addToResult(SearchHits hits,List result,Class tClass){

for (SearchHit hit : hits) {

String sourceAsString = hit.getSourceAsString();

Object object = JsonUtil.parseObject(sourceAsString, tClass);

result.add(object);

}

}

}