赞
踩
阿里云elastic search的简单使用步骤
这里就不详细写如何购买服务器了(需要有elastic search实例和logstash实例)
默认账号 elastic 密码为创建实例是设置的es密码
登录kibana控制台找到开发工具(图标:小扳手),执行下列操作
#创建索引 自定义索引名称test_index
#设置分片数为5 副本数为1
#设置映射 ik_max_word
为细分词 ik_smart
为粗略分词 type字段类型
#type
对应
type | mysql |
---|---|
keyword(不清楚就用keyword) | int,byte,long |
varchar | text |
float | 浮点数或者Bigdecimal |
date | date |
执行创建索引
示例:
#创建索引 PUT /test_index #索引名称 { "settings": { "number_of_shards": 5, #分片数 "number_of_replicas": 1 #副本数 }, "mappings": { "properties" : { "ProId" : { "type" : "keyword" }, "ProName" : { "analyzer" : "ik_max_word", "search_analyzer" : "ik_smart", "type" : "text" }, "Unit" : { "type" : "text" }, "GiveIntegral" : { "type" : "float" }, "ProImg" : { "index" : false, "type" : "keyword" }, "AddDate" : { "format" : "yyyy-MM-dd HH:mm:ss", "type" : "date" }, "categoryId" : { "type" : "keyword" } } } } #设置为非只读模式(如需删除操作) PUT /_settings { "index": { "blocks": { "read_only_allow_delete": "false" } } } #获取索引所有数据及其配置 GET /test_index/_search #根据id删除索引中的数据 DELETE /test_index/_doc/id值 #删除查询到的数据 POST /test_index/_doc/_delete_by_query { "query": { "match_all": {} } } #此处为测试英文分词 POST /test_index/_analyze { "analyzer" : "ik_max_word", "text": "where are you" } #此处为测试中文分词 #粗细分词自己设定 POST /test_index/_analyze { "analyzer" : "ik_smart", "text": "中华人民共和国" }
上传所需要的mysql 驱动jar包
这里mysql驱动jar包可以从自己maven仓库中取出来上传上去
创建管道
input { jdbc { jdbc_driver_class => "com.mysql.jdbc.Driver" jdbc_driver_library => "/ssd/1/share/ls-cn-替换你的实例地址/logstash/current/config/custom/mysql-connector-java-8.0.18.jar" jdbc_connection_string => "jdbc:mysql://xxxx:3306/数据库名称?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowLoadLocalInfile=false&autoDeserialize=false&serverTimezone=CTT&tinyInt1isBit=false&zeroDateTimeBehaviro=convertToNull" jdbc_user => "xxx" jdbc_password => "xxx" jdbc_paging_enabled => "true" jdbc_page_size => "50000" statement => "sql语句一定要加此条件时间字段自定义 where 新增时间字段 >= :sql_last_value OR 修改时间字段 >= :sql_last_value" schedule => "* * * * *" record_last_run => true last_run_metadata_path => "/ssd/1/ls-cn-替换你的实例地址/logstash/data/last_run_metadata_update_time.txt" clean_run => false # 设置为true时,sql_last_value的值是tracking_column的值;设置为false是,sql_last_value的值是上次执行的值。 use_column_value => false } } filter { } output { elasticsearch { hosts => "es-cn-xxxx.elasticsearch.aliyuncs.com:9200" index => "索引名称" user => "elastic" document_id => "%{文档id可以是主键id}" password => "es访问密码" } }
讲上边配置对应填进去
jdbc_driver_library 对应jdbc文件路径
对应依赖
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.5.4</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.5.4</version>
</dependency>
yml配置
es:
elasticsearch:
# hostlist: es-cn-xxxx.public.elasticsearch.aliyuncs.com #本地测试公网(公网需要加白名单)
hostlist: es-cn-xxxx.elasticsearch.aliyuncs.com #线上用内网
user: elastic
password: es访问密码
genteral_store:
index: 索引名称
#source_field: 这里是索引中的字段多个,号分割,与索引创建的字段名称对应
source_field: proid,vipprice,prosum等
config配置类
import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** * @author Administrator * @version 1.0 **/ @Configuration public class ElasticsearchConfig { @Value("${es.elasticsearch.hostlist}") private String hostlist; @Value("${es.elasticsearch.user}") private String user; @Value("${es.elasticsearch.password}") private String password; /** * Java High Level REST Client(本章节以此为例):Elasticsearch Client官方高级客户端。基于低级客户端 * 同步调用方法立即返回一个Response对象。 * 而异步调用方法(方法名以async结尾)依赖于监听,当有请求返回或是错误返回时,该监听会通知到对应的方法继续执行。 */ // @Bean // public RestHighLevelClient restHighLevelClient(){ // //解析hostlist配置信息 // String[] split = hostlist.split(","); // //创建HttpHost数组,其中存放es主机和端口的配置信息 // HttpHost[] httpHostArray = new HttpHost[split.length]; // for(int i=0;i<split.length;i++){ // String item = split[i]; // httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http"); // } // //创建RestHighLevelClient客户端 // return new RestHighLevelClient(RestClient.builder(httpHostArray)); // } //项目主要使用RestHighLevelClient,对于低级的客户端暂时不用 // @Bean // public RestClient restClient(){ // //解析hostlist配置信息 // String[] split = hostlist.split(","); // //创建HttpHost数组,其中存放es主机和端口的配置信息 // HttpHost[] httpHostArray = new HttpHost[split.length]; // for(int i=0;i<split.length;i++){ // String item = split[i]; // httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http"); // } // return RestClient.builder(httpHostArray).build(); // } /** * Java High Level REST Client(本章节以此为例):Elasticsearch Client官方高级客户端。基于低级客户端 * 同步调用方法立即返回一个Response对象。 * 而异步调用方法(方法名以async结尾)依赖于监听,当有请求返回或是错误返回时,该监听会通知到对应的方法继续执行。 */ //阿里云高级客户端配置 @Bean public RestHighLevelClient restHighLevelClient() { // 阿里云ES集群需要basic auth验证。 CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); //访问用户名和密码为您创建阿里云Elasticsearch实例时设置的用户名和密码,也是Kibana控制台的登录用户名和密码。 credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password)); // 通过builder创建rest client,配置http client的HttpClientConfigCallback。 // 单击所创建的Elasticsearch实例ID,在基本信息页面获取公网地址,即为ES集群地址。 RestClientBuilder builder = RestClient.builder(new HttpHost(hostlist, 9200)) .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { @Override public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); } }); //创建RestHighLevelClient客户端 return new RestHighLevelClient(builder); } //项目主要使用RestHighLevelClient,对于低级的客户端暂时不用 /** * Java Low Level REST Client:Elasticsearch Client低级别客户端。它允许通过HTTP请求与ES集群进行通信。 * API本身不负责数据的编码解码,由用户去编码解码。它与所有的ES版本兼容。 * @return */ @Bean public RestClient restClient() { CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password)); RestClient restClient = RestClient.builder(new HttpHost(hostlist, 9200)) .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { @Override public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); } }).build(); return restClient; } }
model层
import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; import java.io.Serializable; /** * 搜索请求参数 */ @Data @NoArgsConstructor @ToString public class GeneralStoreSearchParam{ Integer page;//页码 String keyword; //关键词 Integer sort;//排序方式 String cid; //分类id String cidtwo; //分类id String cidthree; //分类id Boolean min_dis;//从小到大 Boolean max_dis;//从大到小 Double price_max;//价格最高区间 Double price_min;//价格最低区间 String from_where;//商品来源 String jieri_id; String shop_type; //商品类型 }
controller层
import com.zzfeidu.model.GeneralStoreSearchParam; import com.zzfeidu.model.QueryPageResult; import com.zzfeidu.service.ElasticSearchService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; /** * 搜索服务 */ @CrossOrigin(origins = "*", maxAge = 3600) @RestController @RequestMapping("/search") public class ElasticSearchController{ @Autowired private ElasticSearchService elasticSearchService; /** * 根据条件全文检索 * @param generalStoreSearchParam * @return */ @GetMapping("/list") public QueryPageResult search(GeneralStoreSearchParam generalStoreSearchParam) { return elasticSearchService.getlist(generalStoreSearchParam); } }
service实现层
import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.*; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.sort.SortOrder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.IOException; import java.math.BigDecimal; import java.util.*; /** * 搜索服务 */ @Service @Slf4j public class ElasticSearchService { @Autowired RestClient restClient; @Autowired RestHighLevelClient restHighLevelClient; @Value("${es.genteral_store.index}") String index; // @Value("${es.genteral_store.type}") // String doc; @Value("${es.genteral_store.source_field}") String source_field; @Autowired SupplierMapper supplierMapper; @Autowired ProductMapper productMapper; /** * 根据条件进行搜索相关商品 * */ public QueryPageResult<Productdetail> getlist(GeneralStoreSearchParam param) { //防止空指针 if(param == null){ param = new GeneralStoreSearchParam(); } log.info("----进入搜索参数--param="+param); //定义参数集queryResult QueryPageResult<Productdetail> queryResult = new QueryPageResult<>(); List<Productdetail> list = new ArrayList<>(); log.info("----高级搜索-----"); //获取搜索请求对象 SearchRequest general_store = new SearchRequest(index); //指定类型 // general_store.types(doc); //搜索源构建对象 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); //设置源字段过滤,第一个参数结果集包括那些字段,第二个参数标识结果集不包括哪些字段 String[] source_field_array = source_field.split(","); searchSourceBuilder.fetchSource(source_field_array,new String[]{}); //设置分页参数 if(param.getPage() == null || param.getPage() <= 0){ param.setPage(1); } int page = (param.getPage() - 1) * 10; searchSourceBuilder.from(page);//当前页 searchSourceBuilder.size(10);//当前页 //创建布尔搜索对象 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); //multiMatchQuery if(StringUtils.isNotEmpty(param.getKeyword())){ //首先定义一个multiMatch搜索 MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(param.getKeyword(), "proname", "description","keywords","categoryname1","categoryname2","categoryname3") .minimumShouldMatch("100%") .field("kedwords", 10) .field("proname",8) .field("categoryname1",8) .field("categoryname2",8) .field("categoryname3",8); //添加进布尔搜索 boolQueryBuilder.must(multiMatchQueryBuilder); } //分类过滤 if(StringUtils.isNotEmpty(param.getCid())){ //首先定义一个termQuery精确查询 TermQueryBuilder termQuery = QueryBuilders.termQuery("categoryid", param.getCid()); //添加进布尔搜索 boolQueryBuilder.must(termQuery); } //分类过滤 if(StringUtils.isNotEmpty(param.getCidtwo())){ //首先定义一个termQuery精确查询 TermQueryBuilder termQuery = QueryBuilders.termQuery("categorycode", param.getCidtwo()); //添加进布尔搜索 boolQueryBuilder.must(termQuery); } //分类过滤 if(StringUtils.isNotEmpty(param.getCidthree())){ //首先定义一个termQuery精确查询 TermQueryBuilder termQuery = QueryBuilders.termQuery("categorythird", param.getCidthree()); //添加进布尔搜索 boolQueryBuilder.must(termQuery); } //商品类型过滤 if(StringUtils.isNotEmpty(param.getShop_type())){ //首先定义一个termQuery精确查询 TermQueryBuilder termQuery = QueryBuilders.termQuery("shop_type", param.getShop_type()); //添加进布尔搜索 boolQueryBuilder.must(termQuery); } TermQueryBuilder termQuery = QueryBuilders.termQuery("isonsell", 1); boolQueryBuilder.must(termQuery); //添加过虑器 //价格区间过虑 if(param.getPrice_min() != null && param.getPrice_max() != null){ //设置最高价 - 最低价 boolQueryBuilder.filter(QueryBuilders.rangeQuery("balanceprice").gte(param.getPrice_min()).lte(param.getPrice_max())); }else { //设置 最低价 - xxx if(param.getPrice_min() != null){ boolQueryBuilder.filter(QueryBuilders.rangeQuery("balanceprice").gte(param.getPrice_min())); } //设置 0 - 最高价 if(param.getPrice_max() != null){ boolQueryBuilder.filter(QueryBuilders.rangeQuery("balanceprice").gte(0).lte(param.getPrice_max())); } } //获取排序方式 Integer sort = param.getSort(); //设置销量降序 if(sort != null){ if(sort == 1){ searchSourceBuilder.sort("sort", SortOrder.ASC); } if(sort == 2){ searchSourceBuilder.sort("prosum", SortOrder.DESC); } //设置销量升序 if(sort == 3){ searchSourceBuilder.sort("prosum", SortOrder.ASC); } //设置价格升序 if(sort == 4){ searchSourceBuilder.sort("balanceprice", SortOrder.ASC); } //设置价格降序 if(sort == 5){ searchSourceBuilder.sort("balanceprice", SortOrder.DESC); } //链分值降序 if(sort == 6){ searchSourceBuilder.sort("lfz", SortOrder.DESC); } //链分值升序 if(sort == 7){ searchSourceBuilder.sort("lfz", SortOrder.ASC); } //链分值比例降序 if(sort == 8){ searchSourceBuilder.sort("lfzbili", SortOrder.DESC); } //链分值比例升序 if(sort == 9){ searchSourceBuilder.sort("lfzbili", SortOrder.ASC); } } //设置高亮显示 // highlightBuilder.preTags("<span style=\"color:red\">"); // highlightBuilder.postTags("</span>"); //定义高亮-- HighlightBuilder highlightBuilder = new HighlightBuilder(); highlightBuilder.preTags("<text style=\"color: #ffac34;\">"); highlightBuilder.postTags("</text>"); highlightBuilder.fields().add(new HighlightBuilder.Field("proname")); searchSourceBuilder.highlighter(highlightBuilder); searchSourceBuilder.query(boolQueryBuilder); //向搜索请求对象中设置搜索源 general_store.source(searchSourceBuilder); //执行搜索,向es发起http请求 SearchResponse searchResponse = null; try { searchResponse = restHighLevelClient.search(general_store); } catch (IOException e) { e.printStackTrace(); } //搜索结果 SearchHits hits = searchResponse.getHits(); //设置总纪录数 queryResult.setAll_count(hits.getTotalHits() + ""); //匹配度较高的前n个文档 SearchHit[] searchHits = hits.getHits(); for(SearchHit hit : searchHits){ Productdetail productdetail = new Productdetail(); String id = hit.getId();//id //源文档内容 Map<String, Object> sourceAsMap = hit.getSourceAsMap(); String ProId = sourceAsMap.get("proid").toString();//商品id String ProName = (String) sourceAsMap.get("proname");//商品名称 //取出高亮字段--day12 Map<String, HighlightField> highlightFields = hit.getHighlightFields(); if(highlightFields.get("proname")!=null){ HighlightField highlightField = highlightFields.get("proname"); Text[] fragments = highlightField.fragments(); StringBuffer stringBuffer = new StringBuffer(); for(Text text:fragments){ stringBuffer.append(text); } ProName = stringBuffer.toString(); ProName = "<text>" + ProName + "</text>"; } System.out.println("高亮后的ProName:"+ProName); String qiqiuproimgpath = (String) sourceAsMap.get("qiqiuproimgpath");//商品小图片路径 String shop_type = sourceAsMap.get("shop_type").toString(); //商家信息 String VipPrice = sourceAsMap.get("vipprice").toString(); //代理价 String MarketPrice = sourceAsMap.get("marketprice").toString(); //市场价 String BalancePrice = sourceAsMap.get("balanceprice").toString(); //结算价 String IsHit = sourceAsMap.get("ishit").toString(); //商品所属区域 String ConsumeIntegral = sourceAsMap.get("consumeintegral").toString(); //购买此商品消耗的积分 String ProImg = (String) sourceAsMap.get("proimg"); //商品图片路径 String prosum = sourceAsMap.get("prosum").toString(); //销量 String categoryId = sourceAsMap.get("categoryid").toString(); //分类id1 String categoryCode = sourceAsMap.get("categorycode").toString(); //分类id2 String categoryThird= sourceAsMap.get("categorythird").toString(); //分类id3 String SupplierId = sourceAsMap.get("supplierid").toString(); //商家信息 String lfz = "0"; if(sourceAsMap.get("lfz") != null){ lfz = sourceAsMap.get("lfz").toString(); //商家信息 } String lfzbili = sourceAsMap.get("lfzbili").toString(); //商家信息 String suppliername = sourceAsMap.get("suppliername").toString(); //商家信息 productdetail.setId(id); productdetail.setProname(ProName);//商品名称 productdetail.setProid(ProId);//商品id productdetail.setCategoryId(categoryId);//分类id productdetail.setConsumeintegral(ConsumeIntegral);//购买此商品消耗的积分 productdetail.setMarketprice(MarketPrice);//市场价 productdetail.setWholesale_price(MarketPrice);//市场价 if (StringUtils.isNotEmpty(shop_type) && Integer.parseInt(shop_type) != 4) { double vip = StringUtils.isEmpty(VipPrice) ? 0 : Double.parseDouble(VipPrice) * 0.2; productdetail.setVipprice(new BigDecimal(vip).setScale(2,BigDecimal.ROUND_HALF_UP).toString()); }else { productdetail.setVipprice(VipPrice);//代理价 } productdetail.setPronum(prosum);//销量 productdetail.setIshit(IsHit);//商品所属区域 productdetail.setLf(lfz);//商品所属区域 productdetail.setLfzbili(lfzbili);//商品所属区域 productdetail.setShop_type(shop_type);//商品所属区域 productdetail.setCategoryCode(categoryCode);//商品所属区域 productdetail.setCategoryThird(categoryThird);//商品所属区域 productdetail.setSuppliername(suppliername);//商品所属区域 if (!ProImg.startsWith("http")) { productdetail.setProimg(MyConfig.utl + ProImg);//商品图片路径 } else { productdetail.setProimg(ProImg);//商品图片路径 } productdetail.setSupplierid(SupplierId);//商家id productdetail.setWholesale_price(BalancePrice);//批发价 productdetail.setQiqiuproimgpath(qiqiuproimgpath);//商品小图片路径 list.add(productdetail); } // } // queryResult.setData_list(list); HashMap<String, List<Productdetail>> map = new HashMap<>(); map.put("data_list",list); queryResult.setData(map); queryResult.setMsg("查询成功!"); queryResult.setStatus("1"); log.info("----搜索结果 queryResult="+queryResult); return queryResult; } }
附kibana使用文档:https://www.elastic.co/guide/en/kibana/current/index.html
附es映射参考路径:https://blog.csdn.net/u013545439/article/details/102799518
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。