ES(elasticsearch)搜索引擎 结合java 使用_es搜索引擎java使用

ES(elasticsearch)搜索引擎 结合java 使用

官方文档地址：
https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.x/java-rest-high.html

重点概念：
文件索引块是因为使用了倒排索引

倒排索引：
例如  文件a 中有 单词 java php ，文件b中有 java php python ,文件c 中有java , python  
那么 单词对应的文件为:
 java ：a ,b,c       
 php: a,b   
 python : b,c    
  查找php只需要查找a,b文件即可
1
2
3
4
5
6
7

分词器：
例如 this is java ，将该段分为this  ,is,java ，并且记录出现的次数，每次搜索将会算出一个分数，也就是匹配值，越高越匹配。
1
2

一，新建maven 项目，引入依赖

 <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>

        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.16</version>
        </dependency>
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.12.1</version>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.74</version>
        </dependency>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

说明:关键包是spring-boot-starter-data-elasticsearch 其他是辅助包，jsoup 是为了后续从网页上抓取数据使用

二：新建配置类 ElasticsearchConfig

@Configuration
public class ElasticsearchConfig {

    @Bean
    public RestHighLevelClient restHighLevelClient(){
        return new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("your ip", 9200, "http")));
    }
}
1
2
3
4
5
6
7
8
9
10

三：避免不必要的麻烦，将elasticsearch的客户端依赖 设置跟安装的es版本一直
操作方法：
点击pom 文件的 parent 标签 中的artifactId

继续进依赖

可以看到版本

复制出去到pom.xml ，刷新依赖

四：操作api

工具类EsClient ：

@Component
public class EsClient {
    @Resource
    private RestHighLevelClient restHighLevelClient;
    private final String table = "jd_table";

    public String createIndex(V value) throws IOException {
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(table);
        CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest,
                RequestOptions.DEFAULT);
        return createIndexResponse.index();
    }

    /**
     * 保存
     *
     * @param value
     * @return
     * @throws IOException
     */
    public Object save(Object value) throws IOException {
        IndexRequest indexRequest = new IndexRequest(table);
        IndexRequest source = indexRequest.source(JSON.toJSONString(value), XContentType.JSON);
        IndexResponse index = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT);
        return index.getId();
    }

    public boolean existsIndex() throws IOException {
        GetIndexRequest getIndexRequest = new GetIndexRequest(table);
        boolean exists = restHighLevelClient.indices().exists(getIndexRequest, RequestOptions.DEFAULT);
        return exists;

    }

    public Object getDocById(String id) throws IOException {
        GetRequest getRequest = new GetRequest(table, id);
        //不获取_source上下文
        //  getRequest.fetchSourceContext(new FetchSourceContext(false));
        GetResponse documentFields = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
        return documentFields.getSource() != null ? documentFields.getSource() : null;
    }

    public String updateDocById(Object obj, String id) throws IOException {
        UpdateRequest updateRequest = new UpdateRequest(table, id);
        updateRequest.doc(JSON.toJSONString(obj), XContentType.JSON);
        UpdateResponse updateResponse = restHighLevelClient.update(updateRequest, RequestOptions.DEFAULT);
        return updateResponse.getId() != null ? updateResponse.getId() : null;
    }

    public String deletDocById(String id) throws IOException {
        DeleteRequest deleteRequest = new DeleteRequest(table, id);
        DeleteResponse updateResponse = restHighLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
        return updateResponse.getId() != null ? updateResponse.getId() : null;
    }

    public boolean saveBulk(List<Object> values) throws IOException {
        BulkRequest bulkRequest = new BulkRequest(table);
        for (int i = 0; i < values.size(); i++) {
            bulkRequest.add(new IndexRequest(table).id("" + i).source(JSON.toJSONString(values.get(i)),
                    XContentType.JSON));
        }
        bulkRequest.timeout("30s");
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        return !bulk.hasFailures();
    }

    public <T> List<T> searchTerm(String keyword, String fild,Class<T> claz) throws IOException {
        SearchRequest request = new SearchRequest(table);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(fild, keyword);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.query(termQueryBuilder);
        request.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(request, RequestOptions.DEFAULT);
       List<T> lists= new ArrayList<>();
       for(SearchHit hit:search.getHits().getHits()){
           Map<String, Object> sourceAsMap = hit.getSourceAsMap();
           T result = JSONObject.parseObject(JSONObject.toJSONString(sourceAsMap), claz);
           lists.add(result);
       }
       return lists;
    }

    public <T> List<T> searchMatch(String keyword, String fild,Integer form,Integer size, Class<T> claz) throws IOException {
        SearchRequest request = new SearchRequest(table);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.size(size);
        searchSourceBuilder.from(form);
        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery(fild, keyword);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.query(matchQueryBuilder);
        request.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        List<T> lists= new ArrayList<>();
        for(SearchHit hit:search.getHits().getHits()){
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            T result = JSONObject.parseObject(JSONObject.toJSONString(sourceAsMap), claz);
            lists.add(result);
        }
        return lists;
    }


    public <T> List<T> searchMatchHeightLight(String keyword, String fild,Integer form,Integer size, Class<T> claz) throws IOException {
        SearchRequest request = new SearchRequest(table);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //分页
        searchSourceBuilder.size(size);
        searchSourceBuilder.from(form);

        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery(fild, keyword);
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.query(matchQueryBuilder);


        //高亮
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field(fild);
        highlightBuilder.preTags("<span style='colo : red'>");
        highlightBuilder.postTags("</span>");
        searchSourceBuilder.highlighter(highlightBuilder);

        request.source(searchSourceBuilder);

        SearchResponse search = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        List<T> lists= new ArrayList<>();
        for(SearchHit hit:search.getHits().getHits()){

            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            HighlightField fields = highlightFields.get(fild);
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            if(fields!=null){
                Text[] fragments = fields.fragments();
                StringBuilder n_fields=new StringBuilder();
                for(Text text:fragments){
                    n_fields.append(text);
                }
                sourceAsMap.put(fild,n_fields);
            }
            T result = JSONObject.parseObject(JSONObject.toJSONString(sourceAsMap), claz);

            lists.add(result);
        }
        return lists;
    }

    public <T> List<T> searchMatchAll(Class<T> claz) throws IOException {
        SearchRequest request = new SearchRequest(table);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
        searchSourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchSourceBuilder.query(matchAllQueryBuilder);
        request.source(searchSourceBuilder);
        SearchResponse search = restHighLevelClient.search(request, RequestOptions.DEFAULT);
        List<T> lists= new ArrayList<>();
        for(SearchHit hit:search.getHits().getHits()){
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            T result = JSONObject.parseObject(JSONObject.toJSONString(sourceAsMap), claz);
            lists.add(result);
        }
        return lists;
    }
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162

单元测试：

@SpringBootTest
class EsApplicationTests {

    @Autowired
    private EsClient esClient;
    @Test
    /**
     * 创建索引
     */
    void createIndex() throws IOException {
        String index = esClient.createIndex(null);
        System.out.println(esClient.existsIndex());
    }

    @Test
    /**
     * 保存单个
     */
    void testSave() throws IOException {
        Object save = esClient.save(new JdProduct("this is title","this is mg ",2.0d));
        System.out.println(save);
    }

    @Test
    /**
     * 根据id查询文档
     */
    void testGetDocById() throws IOException {
        Object obj = esClient.getDocById("cDSDYHYBbeqLPYSa4sFf");
        System.out.println(obj);
    }

    @Test
    /**
     * 批量插入
     */
    void testbiluk() throws IOException {
        List<JdProduct> objects = new ArrayList<>();
        objects.add(new JdProduct("this is title","this is mg ",2.0d));
        objects.add(new JdProduct("this is title1","this is mg1 ",2.0d));
        objects.add(new JdProduct("this is title2","this is mg2 ",2.0d));
        objects.add(new JdProduct("this is title3","this is mg3 ",2.0d));
        objects.add(new JdProduct("this is title4","this is mg4 ",2.0d));
        objects.add(new JdProduct("this is title5","this is mg5 ",2.0d));
        objects.add(new JdProduct("this is title6","this is mg6 ",2.0d));
        boolean b= esClient.saveBulk(objects);
        System.out.println(b);
    }

    @Test
    /**
     * 精确查询
     */
    void testTreamSearch() throws IOException {

        List<JdProduct> search = esClient.searchTerm("title", "title", JdProduct.class);
        System.out.println(search);
    }

    @Test
    /**
     * 匹配查询
     */
    void testMatchSearch() throws IOException {

        List<JdProduct> search = esClient.searchMatch("title", "title",0,10, JdProduct.class);
        System.out.println(search);
    }

    @Test
    /**
     * 匹配所有
     */
    void testMatchSearchAll() throws IOException {
        List<JdProduct> search = esClient.searchMatchAll(JdProduct.class);
        System.out.println(search);
    }
    @Test
    /**
     * 从京东页面抓取 参数 批量插入 es
     */
    void insertJdFromHtml() throws IOException {
        String urlStr="https://search.jd.com/Search?keyword=java";
        URL url = new URL(urlStr);
        Document document = Jsoup.parse(url, 30000);
        Element j_goodsList = document.getElementById("J_goodsList");
        Elements lis = j_goodsList.getElementsByTag("li");
        List<JdProduct> lists=new ArrayList<>();
        lis.forEach(m->{
            String priceStr = m.getElementsByClass("p-price").eq(0).text();

            Double price = Double.valueOf(priceStr.substring(1,priceStr.length()));
            String title = m.getElementsByClass("p-name").eq(0).text();
            String img = m.getElementsByTag("img").eq(0).attr("data-lazy-img");

            JdProduct jdProduct = new JdProduct(title,img,price);
            lists.add(jdProduct);
        });
        boolean insertFlag = esClient.saveBulk(lists);
        System.out.println(insertFlag);
    }


}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104