赞
踩
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-parent</artifactId> <version>2.5.5</version> <relativePath/> <!-- lookup parent from repository --> </parent> <groupId>com.atlinxi.gulimall</groupId> <artifactId>gulimall-search</artifactId> <version>0.0.1-SNAPSHOT</version> <name>gulimall-search</name> <description>elasticsearch检索服务</description> <properties> <java.version>1.8</java.version> <elasticsearch.version>7.4.2</elasticsearch.version> <spring-cloud.version>2020.0.4</spring-cloud.version> </properties> <dependencies> <dependency> <groupId>com.atlinxi.gulimall</groupId> <artifactId>gulimall-common</artifactId> <version>0.0.1-SNAPSHOT</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> <version>7.4.2</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.47</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> </dependency> </dependencies> <dependencyManagement> <dependencies> <dependency> <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-dependencies</artifactId> <version>${spring-cloud.version}</version> <type>pom</type> <scope>import</scope> </dependency> </dependencies> </dependencyManagement> <build> <plugins> <plugin> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-maven-plugin</artifactId> </plugin> </plugins> </build> </project>
spring.cloud.nacos.discovery.server-addr=127.0.0.1:8848
spring.application.name=gulimall-search
package com.atlinxi.gulimall.search.config; import org.apache.http.HttpHost; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; /** * es配置类,给容器中注入一个RestHighLevelClient */ @Configuration public class GulimallElasticSearchConfig { // 后端访问es的时候,出于安全考虑,可以携带一个请求头 // 现在暂时不用 public static final RequestOptions COMMON_OPTIONS; static { RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); // builder.addHeader("Authorization", "Bearer " + TOKEN); // builder.setHttpAsyncResponseConsumerFactory( // new HttpAsyncResponseConsumerFactory // .HeapBufferedResponseConsumerFactory(30 * 1024 * 1024 * 1024)); COMMON_OPTIONS = builder.build(); } @Bean public RestHighLevelClient esRestClient(){ RestHighLevelClient client = new RestHighLevelClient( RestClient.builder( new HttpHost("192.168.56.10", 9200, "http") // new HttpHost("localhost", 9201, "http") )); return client; } }
package com.atlinxi.gulimall.search; import com.alibaba.fastjson.JSON; import com.atlinxi.gulimall.search.config.GulimallElasticSearchConfig; import lombok.Data; import lombok.ToString; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Avg; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import java.io.IOException; @SpringBootTest class GulimallSearchApplicationTests { @Autowired RestHighLevelClient restHighLevelClient; @Test public void searchData()throws IOException{ // 1. 创建检索请求 SearchRequest searchRequest = new SearchRequest(); // 1.1 指定索引 searchRequest.indices("bank"); // 指定DSL,检索条件 SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // 1.2 构造检索条件 // 所有的函数名都对应原生es DSL语句 // searchSourceBuilder.query(); // searchSourceBuilder.from(); // searchSourceBuilder.size(); // searchSourceBuilder.aggregation(); searchSourceBuilder.query(QueryBuilders.matchQuery("address","mill")); // 按照年龄的值分布进行聚合 TermsAggregationBuilder ageAgg = AggregationBuilders.terms("ageAgg").field("age").size(10); searchSourceBuilder.aggregation(ageAgg); // 计算平均薪资 AvgAggregationBuilder balanceAvg = AggregationBuilders.avg("balanceAvg").field("balance"); searchSourceBuilder.aggregation(balanceAvg); // System.out.println(searchSourceBuilder.toString()); searchRequest.source(searchSourceBuilder); // 2. 执行检索 SearchResponse searchResponse = restHighLevelClient.search(searchRequest, GulimallElasticSearchConfig.COMMON_OPTIONS); // 3. 分析结果 searchResponse // System.out.println(searchResponse.toString()); // 3.1 获取所有查到的数据 SearchHits hits = searchResponse.getHits(); SearchHit[] searchHits = hits.getHits(); for (SearchHit searchHit : searchHits) { // searchHit.getIndex(); String string = searchHit.getSourceAsString(); Account account = JSON.parseObject(string, Account.class); // accountGulimallSearchApplicationTests.Account(account_number=970, balance=19648, firstname=Forbes, lastname=Wallace, age=28, gender=M, address=990 Mill Road, employer=Pheast, email=forbeswallace@pheast.com, city=Lopezo, state=AK) // accountGulimallSearchApplicationTests.Account(account_number=136, balance=45801, firstname=Winnie, lastname=Holland, age=38, gender=M, address=198 Mill Lane, employer=Neteria, email=winnieholland@neteria.com, city=Urie, state=IL) // accountGulimallSearchApplicationTests.Account(account_number=345, balance=9812, firstname=Parker, lastname=Hines, age=38, gender=M, address=715 Mill Avenue, employer=Baluba, email=parkerhines@baluba.com, city=Blackgum, state=KY) // accountGulimallSearchApplicationTests.Account(account_number=472, balance=25571, firstname=Lee, lastname=Long, age=32, gender=F, address=288 Mill Street, employer=Comverges, email=leelong@comverges.com, city=Movico, state=MT) System.out.println("account" + account); } // 3.2 获取这次检索到的分析信息 Aggregations aggregations = searchResponse.getAggregations(); // for (Aggregation aggregation : aggregations.asList()) { // System.out.println(aggregation.getName()); // } Terms ageAgg1 = aggregations.get("ageAgg"); for (Terms.Bucket bucket : ageAgg1.getBuckets()) { String keyAsString = bucket.getKeyAsString(); // 年龄38==>2 // 年龄28==>1 // 年龄32==>1 System.out.println("年龄" + keyAsString + "==>" + bucket.getDocCount()); } Avg balanceAvg1 = aggregations.get("balanceAvg"); // 平均薪资:25208.0 System.out.println("平均薪资:" + balanceAvg1.getValue()); } @ToString @Data static class Account{ private int account_number; private int balance; private String firstname; private String lastname; private int age; private String gender; private String address; private String employer; private String email; private String city; private String state; } /** * 存储数据 */ @Test void indexData() throws IOException { // 添加数据有多种方式,例如hashmap、直接将json粘在这儿 IndexRequest request = new IndexRequest("users"); request.id("1"); // request.source("userName","zhangsan","age",12,"gender","男"); // String jsonString = "{" + // "\"user\":\"kimchy\"," + // "\"postDate\":\"2013-01-30\"," + // "\"message\":\"trying out Elasticsearch\"" + // "}"; // request.source(jsonString, XContentType.JSON); User user = new User(); user.setUserName("zs"); user.setAge(12); user.setGender("man"); String jsonString = JSON.toJSONString(user); request.source(jsonString, XContentType.JSON); // 执行保存/更新操作 IndexResponse index = restHighLevelClient.index(request, GulimallElasticSearchConfig.COMMON_OPTIONS); // 提取有用的响应数据 System.out.println(index); } @Data class User{ private String userName; private String gender; private Integer age; } @Test void contextLoads() { System.out.println(restHighLevelClient); } }
这样的结构比较容易检索
,但是会产生冗余字段
,
多个sku是共享一个spu的,所以以下结构中多个sku的attrs
是冗余的,
假设我们有100万个商品,平均属性有20个(2kb),那么冗余数据有2000m,也就是2g,那么整个商城的商品也就多了2g,加一个内存条就ok了
{
skuId:1,
skuTitle:华为XX,
price:998,
saleCount:99,
attrs:[
{尺寸:5寸},
{cpu:高通},
{分辨率:高清}
]
}
这样的结构,attrs只存了一次,没有了冗余数据,也容易检索
但是这样会有一个极大的问题,
搜索小米
,会出来粮食,手机,电器(很多sku都会包含小米)
,会检索出这些spu所有的规格属性,
假设小米的商品有10000个,涉及到4000个spu,4000个spu对应的所有属性,
esClient查询属性的时候,需要携带4000个spuId,因为是Long型数据,每一个都占8个字节,那么查询一次属性需要传递32kb的数据,
假设并发是1万,数据则是320m,如果面对百万并发,则是32G
最终面对百万并发,光传输4000个spuId,数据就高达32G,其他不说,光网络阻塞就非常长
sku索引{
skuId:1,
spuId:11,
xxxxx
}
attr索引{
spuId:11,
attrs:[
{尺寸:5寸},
{cpu:高通},
{分辨率:高清}
]
}
总之一句话,空间和时间
总是不能二者兼得,第一种浪费空间节省时间,第二种浪费时间节省空间,最终我们选择的是时间
PUT product { "mappings": { "properties": { "skuId": { "type": "long" }, "spuId": { "type": "keyword" }, "skuTitle": { "type": "text", "analyzer": "ik_smart" }, "skuPrice": { "type": "keyword" }, "skuImg": { "type": "keyword", "index": false, "doc_values": false }, "saleCount": { "type": "long" }, "hasStock": { "type": "boolean" }, "hotScore": { "type": "long" }, "brandId": { "type": "long" }, "catalogId": { "type": "long" }, "brandName": { "type": "keyword", "index": false, "doc_values": false }, "brandImg": { "type": "keyword", "index": false, "doc_values": false }, "catalogName": { "type": "keyword", "index": false, "doc_values": false }, "attrs": { "type": "nested", "properties": { "attrId": { "type": "long" }, "attrName": { "type": "keyword", "index": false, "doc_values": false }, "attrValue": { "type": "keyword" } } } } } }
/** * 商品上架 * @param spuId */ @Transactional @Override public void up(Long spuId) { // 1. 组装需要的数据 // 1.1 查出当前spuId对应的所有sku信息 List<SkuInfoEntity> skus = skuInfoService.getSkusBySpuId(spuId); List<Long> skuIdList = skus.stream().map(SkuInfoEntity::getSkuId).collect(Collectors.toList()); // 4. 查询当前sku可以被检索的所有规格属性 List<ProductAttrValueEntity> baseAttrs = attrValueService.baseAttrlistforspu(spuId); List<Long> attrIds = baseAttrs.stream().map(attr -> { return attr.getAttrId(); }).collect(Collectors.toList()); List<Long> searchAttrIds = attrService.selectSearchAttrs(attrIds); Set<Long> idSet = new HashSet<>(searchAttrIds); List<SkuEsModel.Attrs> attrsList = baseAttrs.stream().filter(item -> { return idSet.contains(item.getAttrId()); }).map(item -> { SkuEsModel.Attrs attrs1 = new SkuEsModel.Attrs(); BeanUtils.copyProperties(item, attrs1); return attrs1; } ).collect(Collectors.toList()); Map<Long, Boolean> stockMap = null; try { // 1. 发送远程调用,库存系统查询是否有库存 R skusHasStock = wareFeignService.getSkusHasStock(skuIdList); // TypeReference 权限修饰符是 protected,所以我们要写成内部类的方式 List<SkuHasStockVo> data = skusHasStock.getData("data",new TypeReference<List<SkuHasStockVo>>(){}); stockMap = data.stream().collect(Collectors.toMap(SkuHasStockVo::getSkuId, item -> item.getHasStock())); }catch (Exception e){ log.error("库存服务查询异常:",e); } // 1.2 封装每个sku的信息 Map<Long, Boolean> finalstockMap = stockMap; List<SkuEsModel> upProducts = skus.stream().map( sku -> { SkuEsModel skuEsModel = new SkuEsModel(); BeanUtils.copyProperties(sku, skuEsModel); // skuPrice,skuImg,hasStock,hotScore,brandName,brandImg,catalogName // Attrs attrId,attrName,attrValue skuEsModel.setSkuPrice(sku.getPrice()); skuEsModel.setSkuImg(sku.getSkuDefaultImg()); // 设置库存信息 skuEsModel.setHasStock(finalstockMap == null ? true : finalstockMap.get(sku.getSkuId())); // todo 2. 热度评分 先默认设置0 但实际情况应该是后台可控的,比较复杂的操作 skuEsModel.setHotScore(0L); // 3. 查询品牌和分类的名字信息 BrandEntity brand = brandService.getById(skuEsModel.getBrandId()); skuEsModel.setBrandName(brand.getName()); skuEsModel.setBrandImg(brand.getLogo()); CategoryEntity categoryEntity = categoryService.getById(skuEsModel.getCatalogId()); skuEsModel.setCatalogName(categoryEntity.getName()); // 设置检索属性 skuEsModel.setAttrs(attrsList); return skuEsModel; } ).collect(Collectors.toList()); // todo 5. 将数据发送给es进行保存 gulimall-search R r = searchFeignService.productStatusUp(upProducts); // 远程调用成功 if (r.getCode()==0){ // 修改当前spu状态 baseMapper.updateSpuStatus(spuId, ProductConstant.StatusEnum.SPU_UP.getCode()); }else { // todo 重复调用,接口幂等性,重试机制 } }
spuInfoServiceImpl是总的流程,下面都是它调用到的函数
// SkuInfoServiceImpl public List<SkuInfoEntity> getSkusBySpuId(Long spuId) { List<SkuInfoEntity> list = this.list(new QueryWrapper<SkuInfoEntity>().eq("spu_id",spuId)); return list; } // ProductAttrValueServiceImpl public List<ProductAttrValueEntity> baseAttrlistforspu(Long spuId) { List<ProductAttrValueEntity> entities = this.baseMapper.selectList(new QueryWrapper<ProductAttrValueEntity>().eq("spu_id", spuId)); return entities; } // AttrServiceImpl @Override public List<Long> selectSearchAttrs(List<Long> attrIds) { return baseMapper.selectSearchAttrIds(attrIds); } <select id="selectSearchAttrIds" resultType="java.lang.Long"> select attr_id from pms_attr where attr_id in <foreach collection="attrIds" separator="," item="id" open="(" close=")"> #{id} </foreach> and search_type = 1 </select> // WareSkuServiceImpl public List<SkuHasStockVo> getSkusHasStock(List<Long> skuIds) { List<SkuHasStockVo> collect = skuIds.stream().map(skuId -> { SkuHasStockVo vo = new SkuHasStockVo(); // 查询当前sku的总库存量 Long count = this.baseMapper.getSkuStock(skuId); vo.setSkuId(skuId); vo.setHasStock(count==null?false:count>0); return vo; }).collect(Collectors.toList()); return collect; } // SpuInfoDao.xml <update id="updateSpuStatus"> update pms_spu_info set publish_status=#{code},update_time=NOW() where id = #{spuId} </update>
package com.atlinxi.gulimall.search.controller; import com.atlinxi.common.exception.BizCodeEnume; import com.atlinxi.common.to.es.SkuEsModel; import com.atlinxi.common.utils.R; import com.atlinxi.gulimall.search.service.ProductSaveService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.io.IOException; import java.util.List; @Slf4j @RestController @RequestMapping("/search/save") public class ElasticSaveController { @Autowired private ProductSaveService productSaveService; /** * 上架商品 */ @PostMapping("/product") public R productStatusUp(@RequestBody List<SkuEsModel> skuEsModels){ boolean b = false; try { b = productSaveService.productStatusUp(skuEsModels); } catch (IOException e) { log.error("ElasticSaveController商品上架错误:{}",e); return R.error(BizCodeEnume.PRODUCT_UP_EXCEPTION.getCode(), BizCodeEnume.PRODUCT_UP_EXCEPTION.getMessage()); } if (!b){ return R.ok(); }else { return R.error(); } } } package com.atlinxi.gulimall.search.service.impl; import com.alibaba.fastjson.JSON; import com.atlinxi.common.to.es.SkuEsModel; import com.atlinxi.gulimall.search.config.GulimallElasticSearchConfig; import com.atlinxi.gulimall.search.constant.EsConstant; import com.atlinxi.gulimall.search.service.ProductSaveService; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.common.xcontent.XContentType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @Service @Slf4j public class ProductSaveServiceImpl implements ProductSaveService { @Autowired private RestHighLevelClient restHighLevelClient; /** * * @param skuEsModels * @return */ @Override public boolean productStatusUp(List<SkuEsModel> skuEsModels) throws IOException { // 保存到es // 1. 建立索引 product 建立好映射关系(kibana操作) // 2. 给es中保存这些数据 BulkRequest bulkRequest = new BulkRequest(); for (SkuEsModel skuEsModel : skuEsModels) { IndexRequest indexRequest = new IndexRequest(EsConstant.Product_INDEX); indexRequest.id(skuEsModel.getSkuId().toString()); String s = JSON.toJSONString(skuEsModel); indexRequest.source(s, XContentType.JSON); bulkRequest.add(indexRequest); } BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, GulimallElasticSearchConfig.COMMON_OPTIONS); // todo 如果批量错误,处理错误 boolean b = bulk.hasFailures(); List<String> collect = Arrays.stream(bulk.getItems()).map(item -> { return item.getId(); }).collect(Collectors.toList()); log.info("商品上架成功:{}",collect); return b; } }
/** * Copyright (c) 2016-2019 人人开源 All rights reserved. * * https://www.renren.io * * 版权所有,侵权必究! */ package com.atlinxi.common.utils; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; import org.apache.http.HttpStatus; import java.util.HashMap; import java.util.Map; /** * 返回数据 * * @author Mark sunlightcs@gmail.com * * * 老师是用泛型的方式封装的data, * 在feign接口中返回泛型类时,由于java的泛型机制,在实例化之前无法得到具体的类型 , * 因此,虽然服务提供方返回的是具体实例的数据,但是在客户端decode时,无法转化为具体的类。 * * 上面的话看不太懂,翻译成人话就是,feign在被远程调用返回结果的时候,泛型是null * * 因为R继承了HashMap,我们写的所有私有属性都没用,只能存键值对,具体原因未知, * * public class R<T> extends HashMap<String, Object> { * private static final long serialVersionUID = 1L; * * private T data; * * public T getData() { * return this.data; * } * * * public void setData(T data) { * this.data = data; * } */ public class R extends HashMap<String, Object> { private static final long serialVersionUID = 1L; public R setData(Object data){ put("data",data); return this; } //利用fastjson进行反序列化 public <T> T getData(TypeReference<T> typeReference) { Object data = get("data"); //默认是map String jsonString = JSON.toJSONString(data); T t = JSON.parseObject(jsonString, typeReference); return t; } //利用fastjson进行反序列化 public <T> T getData(String key,TypeReference<T> typeReference) { Object data = get(key); //默认是map String jsonString = JSON.toJSONString(data); T t = JSON.parseObject(jsonString, typeReference); return t; } public R() { put("code", 0); put("msg", "success"); } public static R error() { return error(HttpStatus.SC_INTERNAL_SERVER_ERROR, "未知异常,请联系管理员"); } public static R error(String msg) { return error(HttpStatus.SC_INTERNAL_SERVER_ERROR, msg); } public static R error(int code, String msg) { R r = new R(); r.put("code", code); r.put("msg", msg); return r; } public static R ok(String msg) { R r = new R(); r.put("msg", msg); return r; } public static R ok(Map<String, Object> map) { R r = new R(); r.putAll(map); return r; } public static R ok() { return new R(); } public R put(String key, Object value) { super.put(key, value); return this; } public int getCode(){ return (Integer) this.get("code"); } }
package com.atlinxi.common.constant; public class ProductConstant { public enum AttrEnum{ ATTR_TYPE_BASE(1,"基本属性"),ATTR_TYPE_SALE(0,"销售属性"); private int code; private String msg; AttrEnum(int code, String msg) { this.code = code; this.msg = msg; } public int getCode() { return code; } public void setCode(int code) { this.code = code; } public String getMsg() { return msg; } public void setMsg(String msg) { this.msg = msg; } } public enum StatusEnum{ NEW_SPU(0,"新建"),SPU_UP(1,"商品上架"), SPU_DOWN(2,"商品下架"); private int code; private String msg; StatusEnum(int code, String msg) { this.code = code; this.msg = msg; } public int getCode() { return code; } public void setCode(int code) { this.code = code; } public String getMsg() { return msg; } public void setMsg(String msg) { this.msg = msg; } } }
但是她总觉得怪怪的,李国华的眼睛里有一种研究的意味。很久以后,伊纹才会知道,李国华想要在她脸上预习思琪将来的表情。
房思琪的初恋乐园
林奕含
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。