devbox
代码探险家
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

Milvus 基本操作_milvus milvusclient.search(searchparam) 评分

作者：代码探险家 | 2024-07-14 05:48:18

赞

踩

milvus milvusclient.search(searchparam) 评分

1、maven 依赖

  1. <dependency>
  2.             <groupId>io.milvus</groupId>
  3.             <artifactId>milvus-sdk-java</artifactId>
  4.             <version>2.3.3</version>
  5.             <exclusions>
  6.                 <exclusion>
  7.                     <groupId>org.slf4j</groupId>
  8.                     <artifactId>slf4j-api</artifactId>
  9.                 </exclusion>
  10.                 <exclusion>
  11.                     <groupId>org.apache.logging.log4j</groupId>
  12.                     <artifactId>log4j-slf4j-impl</artifactId>
  13.                 </exclusion>
  14.             </exclusions>
  15.         </dependency>

2、MivusService 封装了 基本操作

  1. @Service
  2. @Slf4j
  3. public class MivusService {
  4.  
  5.  
  6.     @Autowired
  7.     MilvusServiceClient milvusClient;
  8.  
  9.     private String clientId;
  10.  
  11.     /**
  12.      * 同步搜索milvus
  13.      * @param collectionName 表名
  14.      * @param vectors 查询向量
  15.      * @param topK 最相似的向量个数
  16.      * @return
  17.      */
  18.     public List<Long> search(String collectionName, List<List<Float>> vectors, Integer topK) {
  20.         Assert.notNull(collectionName, "collectionName  is null");
  21.         Assert.notNull(vectors, "vectors is null");
  22.         Assert.notEmpty(vectors, "vectors is empty");
  23.         Assert.notNull(topK, "topK is null");
  24.         int nprobeVectorSize = vectors.get(0).size();
  25.         String paramsInJson = "{\"nprobe\": " + nprobeVectorSize + "}";
  26.         SearchParam searchParam =
  27.                 SearchParam.newBuilder().withCollectionName(collectionName)
  28.                         .withParams(paramsInJson)
  29.                         .withMetricType(MetricType.L2)
  30.                         .withVectors(vectors)
  31.                         .withVectorFieldName("embeddings")
  32.                         .withTopK(topK)
  33.                         .build();
  34.  
  35.         R<SearchResults> searchResultsR = milvusClient.search(searchParam);
  36.         SearchResults searchResultsRData = searchResultsR.getData();
  37.         List<Long> topksList = searchResultsRData.getResults().getIds().getIntId().getDataList();
  38.         return topksList;
  39.     }
  40.  
  41.     /**
  42.      * 同步搜索milvus
  43.      * @param collectionName 表名
  44.      * @param vectors 查询向量
  45.      * @param topK 最相似的向量个数
  46.      * @return
  47.      */
  48.     public List<Long> search1(String collectionName, List<List<Float>> vectors, Integer topK) {
  50.         Assert.notNull(collectionName, "collectionName  is null");
  51.         Assert.notNull(vectors, "vectors is null");
  52.         Assert.notEmpty(vectors, "vectors is empty");
  53.         Assert.notNull(topK, "topK is null");
  54.         int nprobeVectorSize = vectors.get(0).size();
  55.         String paramsInJson = "{\"nprobe\": " + nprobeVectorSize + "}";
  56.         SearchParam searchParam =
  57.                 SearchParam.newBuilder().withCollectionName(collectionName)
  58.                         .withParams(paramsInJson)
  59.                         .withMetricType(MetricType.IP)
  60.                         .withVectors(vectors)
  61.                         .withVectorFieldName("embedding")
  62.                         .withTopK(topK)
  63.                         .build();
  64.  
  65.         R<SearchResults> searchResultsR = milvusClient.search(searchParam);
  66.         SearchResults searchResultsRData = searchResultsR.getData();
  67.         List<Long> topksList = searchResultsRData.getResults().getIds().getIntId().getDataList();
  68.         return topksList;
  69.     }
  70.  
  71.  
  72.     /**
  73.      * 同步搜索milvus，增加过滤条件搜索
  74.      *
  75.      * @param collectionName 表名
  76.      * @param vectors 查询向量
  77.      * @param topK 最相似的向量个数
  78.      * @param exp 过滤条件：status=1
  79.      * @return
  80.      */
  81.     public List<Long> search2(String collectionName, List<List<Float>> vectors, Integer topK, String exp) {
  82.         Assert.notNull(collectionName, "collectionName  is null");
  83.         Assert.notNull(vectors, "vectors is null");
  84.         Assert.notEmpty(vectors, "vectors is empty");
  85.         Assert.notNull(topK, "topK is null");
  86.         Assert.notNull(exp, "exp is null");
  87.         int nprobeVectorSize = vectors.get(0).size();
  88.         String paramsInJson = "{\"nprobe\": " + nprobeVectorSize + "}";
  89.         SearchParam searchParam =
  90.                 SearchParam.newBuilder().withCollectionName(collectionName)
  91.                         .withParams(paramsInJson)
  92.                         .withMetricType(MetricType.IP)
  93.                         .withVectors(vectors)
  94.                         .withExpr(exp)
  95.                         .withVectorFieldName("embedding")
  96.                         .withTopK(topK)
  97.                         .build();
  98.  
  99.         R<SearchResults> searchResultsR = milvusClient.search(searchParam);
  100.         SearchResults searchResultsRData = searchResultsR.getData();
  101.         List<Long> topksList = searchResultsRData.getResults().getIds().getIntId().getDataList();
  102.         return topksList;
  103.     }
  104.  
  105.     /**
  106.      * 异步搜索milvus
  107.      *
  108.      * @param collectionName 表名
  109.      * @param vectors 查询向量
  110.      * @param partitionList 最相似的向量个数
  111.      * @param topK
  112.      * @return
  113.      */
  114.     public List<Long> searchAsync(String collectionName, List<List<Float>> vectors,
  115.                                   List<String> partitionList, Integer topK) throws ExecutionException, InterruptedException {
  116.  
  117.         Assert.notNull(collectionName, "collectionName  is null");
  118.         Assert.notNull(vectors, "vectors is null");
  119.         Assert.notEmpty(vectors, "vectors is empty");
  120.         Assert.notNull(partitionList, "partitionList is null");
  121.         Assert.notEmpty(partitionList, "partitionList is empty");
  122.         Assert.notNull(topK, "topK is null");
  123.         int nprobeVectorSize = vectors.get(0).size();
  124.         String paramsInJson = "{\"nprobe\": " + nprobeVectorSize + "}";
  125.         SearchParam searchParam =
  126.                 SearchParam.newBuilder().withCollectionName(collectionName)
  127.                         .withParams(paramsInJson)
  128.                         .withVectors(vectors)
  129.                         .withTopK(topK)
  130.                         .withPartitionNames(partitionList)
  131.                         .build();
  132.         ListenableFuture<R<SearchResults>> listenableFuture = milvusClient.searchAsync(searchParam);
  134.         List<Long> resultIdsList = listenableFuture.get().getData().getResults().getTopksList();
  135.  
  136.         return resultIdsList;
  137.     }
  138.  
  139.     /**
  140.      * 获取分区集合
  141.      * @param collectionName 表名
  142.      * @return
  143.      */
  144.     public List<String> getPartitionsList(String collectionName) {
  145.         Assert.notNull(collectionName, "collectionName  is null");
  146.         ShowPartitionsParam searchParam = ShowPartitionsParam.newBuilder().withCollectionName(collectionName).build();
  147.         List<ByteString> byteStrings = milvusClient.showPartitions(searchParam).getData().getPartitionNamesList().asByteStringList();
  148.         List<String> partitionList = Lists.newLinkedList();
  149.         byteStrings.forEach(s -> {
  150.             partitionList.add(s.toStringUtf8());
  151.         });
  152.         return partitionList;
  153.     }
  154.  
  155.  
  156.  
  157.     public void loadCollection(String collectionName) {
  158.         LoadCollectionParam loadCollectionParam = LoadCollectionParam.newBuilder()
  159.                 .withCollectionName(collectionName)
  160.                 .build();
  161.         R<RpcStatus> response = milvusClient.loadCollection(loadCollectionParam);
  162.         log.info("loadCollection {} is {}", collectionName, response.getData().getMsg());
  163.     }
  164.  
  165.  
  166.     public void releaseCollection(String collectionName) {
  167.         ReleaseCollectionParam param = ReleaseCollectionParam.newBuilder()
  168.                 .withCollectionName(collectionName)
  169.                 .build();
  170.         R<RpcStatus> response = milvusClient.releaseCollection(param);
  171.         log.info("releaseCollection {} is {}", collectionName, response.getData().getMsg());
  172.     }
  173.  
  174.  
  175.     public void loadPartitions(String collectionName, List<String> partitionsName) {
  176.         LoadPartitionsParam build = LoadPartitionsParam.newBuilder()
  177.                 .withCollectionName(collectionName)
  178.                 .withPartitionNames(partitionsName)
  179.                 .build();
  180.         R<RpcStatus> rpcStatusR = milvusClient.loadPartitions(build);
  181.         log.info("loadPartitions {} is {}", partitionsName, rpcStatusR.getData().getMsg());
  182.     }
  183.  
  184.  
  185.     public void releasePartitions(String collectionName, List<String> partitionsName) {
  186.         ReleasePartitionsParam build = ReleasePartitionsParam.newBuilder()
  187.                 .withCollectionName(collectionName)
  188.                 .withPartitionNames(partitionsName)
  189.                 .build();
  190.         R<RpcStatus> rpcStatusR = milvusClient.releasePartitions(build);
  191.         log.info("releasePartition {} is {}", collectionName, rpcStatusR.getData().getMsg());
  192.     }
  193.  
  194.  
  195.     public boolean isExitCollection(String collectionName) {
  196.         HasCollectionParam hasCollectionParam = HasCollectionParam.newBuilder()
  197.                 .withCollectionName(collectionName)
  198.                 .build();
  199.         R<Boolean> response = milvusClient.hasCollection(hasCollectionParam);
  200.         Boolean isExists = response.getData();
  201.         log.info("collection {} is exists: {}", collectionName, isExists);
  202.         return isExists;
  203.     }
  204.  
  205.  
  206.     public Boolean creatCollection(String collectionName) {
  207.         // 主键字段
  208.         FieldType fieldType1 = FieldType.newBuilder()
  209.                 .withName(Content.Field.ID)
  210.                 .withDescription("primary key")
  211.                 .withDataType(DataType.Int64)
  212.                 .withPrimaryKey(true)
  213.                 .withAutoID(true)
  214.                 .build();
  215.         // 文本字段
  216.         FieldType fieldType2 = FieldType.newBuilder()
  217.                 .withName(Content.Field.CONTENT)
  218.                 .withDataType(DataType.VarChar)
  219.                 .withMaxLength(Content.MAX_LENGTH)
  220.                 .build();
  221.         // 向量字段
  222.         FieldType fieldType3 = FieldType.newBuilder()
  223.                 .withName(Content.Field.CONTENT_VECTOR)
  224.                 .withDataType(DataType.FloatVector)
  225.                 .withDimension(Content.FEATURE_DIM)
  226.                 .build();
  227.         // 创建collection
  228.         CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
  229.                 .withCollectionName(collectionName)
  230.                 .withDescription("Schema of Content")
  231.                 .withShardsNum(Content.SHARDS_NUM)
  232.                 .addFieldType(fieldType1)
  233.                 .addFieldType(fieldType2)
  234.                 .addFieldType(fieldType3)
  235.                 .build();
  236.         R<RpcStatus> response = milvusClient.createCollection(createCollectionReq);
  237.  
  238.         log.info("collection: {} is created ? status: = {}", collectionName, response.getData().getMsg());
  239.         return response.getData().getMsg().equals("Success");
  240.     }
  241.  
  242.  
  243.     public Boolean dropCollection(String collectionName) {
  244.         DropCollectionParam book = DropCollectionParam.newBuilder()
  245.                 .withCollectionName(collectionName)
  246.                 .build();
  247.         R<RpcStatus> response = milvusClient.dropCollection(book);
  248.         return response.getData().getMsg().equals("Success");
  249.     }
  250.  
  251.  
  252.     public void createPartition(String collectionName, String partitionName) {
  253.         CreatePartitionParam param = CreatePartitionParam.newBuilder()
  254.                 .withCollectionName(collectionName)
  255.                 .withPartitionName(partitionName)
  256.                 .build();
  257.         R<RpcStatus> partition = milvusClient.createPartition(param);
  258.         String msg = partition.getData().getMsg();
  259.         log.info("create partition: {} in collection: {} is: {}", partition, collectionName, msg);
  260.     }
  261.  
  262.  
  263.     public Boolean createIndex(String collectionName) {
  264.         // IndexType
  265.         final IndexType INDEX_TYPE = IndexType.IVF_FLAT;
  266.         // ExtraParam 建议值为 4 × sqrt(n), 其中 n 指 segment 最多包含的 entity 条数。
  267.         final String INDEX_PARAM = "{\"nlist\":16384}";
  268.         long startIndexTime = System.currentTimeMillis();
  269.         R<RpcStatus> response = milvusClient.createIndex(CreateIndexParam.newBuilder()
  270.                 .withCollectionName(collectionName)
  271.                 .withIndexName(Content.CONTENT_INDEX)
  272.                 .withFieldName(Content.Field.CONTENT_VECTOR)
  273.                 .withMetricType(MetricType.L2)
  274.                 .withIndexType(INDEX_TYPE)
  275.                 .withExtraParam(INDEX_PARAM)
  276.                 .withSyncMode(Boolean.TRUE)
  277.                 .withSyncWaitingInterval(500L)
  278.                 .withSyncWaitingTimeout(30L)
  279.                 .build());
  280.         long endIndexTime = System.currentTimeMillis();
  281.         log.info("Succeed in " + (endIndexTime - startIndexTime) / 1000.00 + " seconds!");
  282.         log.info("createIndex --->>> {} ", response.toString());
  283.         GetIndexBuildProgressParam build = GetIndexBuildProgressParam.newBuilder()
  284.                 .withCollectionName(collectionName)
  285.                 .build();
  286.         R<GetIndexBuildProgressResponse> idnexResp = milvusClient.getIndexBuildProgress(build);
  287.         log.info("getIndexBuildProgress --->>> {}", idnexResp.getStatus());
  288.  
  289.         return response.getData().getMsg().equals("Success");
  290.     }
  291.  
  292.  
  293.     public ReplyMsg insert(String collectionName, List<InsertParam.Field> fields) {
  294.         InsertParam insertParam = InsertParam.newBuilder()
  295.                 .withCollectionName(collectionName)
  296.                 .withFields(fields)
  297.                 .build();
  298.         R<MutationResult> mutationResultR = milvusClient.insert(insertParam);
  299.  
  300.         log.info("Flushing...");
  301.         long startFlushTime = System.currentTimeMillis();
  302.         milvusClient.flush(FlushParam.newBuilder()
  303.                 .withCollectionNames(Collections.singletonList(collectionName))
  304.                 .withSyncFlush(true)
  305.                 .withSyncFlushWaitingInterval(50L)
  306.                 .withSyncFlushWaitingTimeout(30L)
  307.                 .build());
  308.         long endFlushTime = System.currentTimeMillis();
  309.         log.info("Succeed in " + (endFlushTime - startFlushTime) / 1000.00 + " seconds!");
  310.  
  311.         if (mutationResultR.getStatus() == 0){
  312.             long insertCnt = mutationResultR.getData().getInsertCnt();
  313.             log.info("Successfully! Total number of entities inserted: {} ", insertCnt);
  314.             return ReplyMsg.ofSuccess("success", insertCnt);
  315.         }
  316.         log.error("InsertRequest failed!");
  317.         return ReplyMsg.ofErrorMsg("InsertRequest failed!");
  318.     }
  319.  
  320.  
  321.     public List<List<SearchResultVo>> searchTopKSimilarity(SearchParamVo searchParamVo) {
  322.         log.info("------search TopK Similarity------");
  323.         SearchParam searchParam = SearchParam.newBuilder()
  324.                 .withCollectionName(searchParamVo.getCollectionName())
  325.                 .withMetricType(MetricType.L2)
  326.                 .withOutFields(searchParamVo.getOutputFields())
  327.                 .withTopK(searchParamVo.getTopK())
  328.                 .withVectors(searchParamVo.getQueryVectors())
  329.                 .withVectorFieldName(Content.Field.CONTENT_VECTOR)
  330.                 .withParams(searchParamVo.getParams())
  331.                 .build();
  332.  
  333.         R<SearchResults> respSearch = milvusClient.search(searchParam);
  334.         if (respSearch.getData() == null) {
  335.             return null;
  336.         }
  337.         log.info("------ process query results ------");
  338.         SearchResultsWrapper wrapper = new SearchResultsWrapper(respSearch.getData().getResults());
  339.         List<List<SearchResultVo>> result = new ArrayList<>();
  340.         for (int i = 0; i < searchParamVo.getQueryVectors().size(); ++i) {
  341.             List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(i);
  342.             List<QueryResultsWrapper.RowRecord> rowRecords = wrapper.getRowRecords();
  343.             List<SearchResultVo> list = new ArrayList<>();
  344.             for (int j = 0; j < scores.size(); ++j) {
  345.                 SearchResultsWrapper.IDScore score = scores.get(j);
  346.                 QueryResultsWrapper.RowRecord rowRecord = rowRecords.get(j);
  347.                 long longID = score.getLongID();
  348.                 float distance = score.getScore();
  349.                 String content = (String) rowRecord.get(searchParamVo.getOutputFields().get(0));
  350.                 log.info("Top " + j + " ID:" + longID + " Distance:" + distance);
  351.                 log.info("Content: " + content);
  352.                 list.add(SearchResultVo.builder().id(longID).score(distance).conent(content).build());
  353.             }
  354.             result.add(list);
  355.         }
  356.         log.info("Successfully!");
  357.  
  358.         return result;
  359.     }
  360.  
  361.     public Boolean creatCollectionERP(String collectionName) {
  362.         // 主键字段
  363.         FieldType fieldType1 = FieldType.newBuilder()
  364.                 .withName(Content.Field.ID)
  365.                 .withDescription("primary key")
  366.                 .withDataType(DataType.Int64)
  367.                 .withPrimaryKey(true)
  368.                 .withAutoID(true)
  369.                 .build();
  370.         // 文本字段
  371.         FieldType fieldType2 = FieldType.newBuilder()
  372.                 .withName(Content.Field.CONTENT)
  373.                 .withDataType(DataType.VarChar)
  374.                 .withMaxLength(Content.MAX_LENGTH)
  375.                 .build();
  376.         // 向量字段
  377.         FieldType fieldType3 = FieldType.newBuilder()
  378.                 .withName(Content.Field.CONTENT_VECTOR)
  379.                 .withDataType(DataType.FloatVector)
  380.                 .withDimension(Content.FEATURE_DIM)
  381.                 .build();
  382.         FieldType fieldType4 = FieldType.newBuilder()
  383.                 .withName(Content.Field.CONTENT_ANSWER)
  384.                 .withDataType(DataType.VarChar)
  385.                 .withMaxLength(Content.MAX_LENGTH)
  386.                 .build();
  387.         FieldType fieldType5 = FieldType.newBuilder()
  388.                 .withName(Content.Field.TITLE)
  389.                 .withDataType(DataType.VarChar)
  390.                 .withMaxLength(Content.MAX_LENGTH)
  391.                 .build();
  392.         FieldType fieldType6 = FieldType.newBuilder()
  393.                 .withName(Content.Field.PARAM)
  394.                 .withDataType(DataType.VarChar)
  395.                 .withMaxLength(Content.MAX_LENGTH)
  396.                 .build();
  397.         FieldType fieldType7 = FieldType.newBuilder()
  398.                 .withName(Content.Field.TYPE)
  399.                 .withDataType(DataType.VarChar)
  400.                 .withMaxLength(Content.MAX_LENGTH)
  401.                 .build();
  402.         // 创建collection
  403.         CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
  404.                 .withCollectionName(collectionName)
  405.                 .withDescription("Schema of Content ERP")
  406.                 .withShardsNum(Content.SHARDS_NUM)
  407.                 .addFieldType(fieldType1)
  408.                 .addFieldType(fieldType2)
  409.                 .addFieldType(fieldType3)
  410.                 .addFieldType(fieldType4)
  411.                 .addFieldType(fieldType5)
  412.                 .addFieldType(fieldType6)
  413.                 .addFieldType(fieldType7)
  414.                 .build();
  415.         R<RpcStatus> response = milvusClient.createCollection(createCollectionReq);
  416.  
  417.         log.info("collection: {} is created ? status: = {}", collectionName, response.getData().getMsg());
  418.         return response.getData().getMsg().equals("Success");
  419.     }
  420.  
  421.     public Boolean creatCollectionERPCLIP(String collectionName) {
  422.         // 主键字段
  423.         FieldType fieldType1 = FieldType.newBuilder()
  424.                 .withName(Content.Field.ID)
  425.                 .withDescription("primary key")
  426.                 .withDataType(DataType.Int64)
  427.                 .withPrimaryKey(true)
  428.                 .withAutoID(true)
  429.                 .build();
  430.         // 文本字段
  431.         FieldType fieldType2 = FieldType.newBuilder()
  432.                 .withName(Content.Field.CONTENT)
  433.                 .withDataType(DataType.VarChar)
  434.                 .withMaxLength(Content.MAX_LENGTH)
  435.                 .build();
  436.         // 向量字段
  437.         FieldType fieldType3 = FieldType.newBuilder()
  438.                 .withName(Content.Field.CONTENT_VECTOR)
  439.                 .withDataType(DataType.FloatVector)
  440.                 .withDimension(Content.FEATURE_DIM_CLIP)
  441.                 .build();
  442.         FieldType fieldType4 = FieldType.newBuilder()
  443.                 .withName(Content.Field.CONTENT_ANSWER)
  444.                 .withDataType(DataType.VarChar)
  445.                 .withMaxLength(Content.MAX_LENGTH)
  446.                 .build();
  447.         FieldType fieldType5 = FieldType.newBuilder()
  448.                 .withName(Content.Field.TITLE)
  449.                 .withDataType(DataType.VarChar)
  450.                 .withMaxLength(Content.MAX_LENGTH)
  451.                 .build();
  452.         FieldType fieldType6 = FieldType.newBuilder()
  453.                 .withName(Content.Field.PARAM)
  454.                 .withDataType(DataType.VarChar)
  455.                 .withMaxLength(Content.MAX_LENGTH)
  456.                 .build();
  457.         FieldType fieldType7 = FieldType.newBuilder()
  458.                 .withName(Content.Field.TYPE)
  459.                 .withDataType(DataType.VarChar)
  460.                 .withMaxLength(Content.MAX_LENGTH)
  461.                 .build();
  462.         FieldType fieldType8 = FieldType.newBuilder()
  463.                 .withName(Content.Field.LABEL)
  464.                 .withDataType(DataType.VarChar)
  465.                 .withMaxLength(Content.MAX_LENGTH)
  466.                 .build();
  467.         // 创建collection
  468.         CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
  469.                 .withCollectionName(collectionName)
  470.                 .withDescription("Schema of Content ERP")
  471.                 .withShardsNum(Content.SHARDS_NUM)
  472.                 .addFieldType(fieldType1)
  473.                 .addFieldType(fieldType2)
  474.                 .addFieldType(fieldType3)
  475.                 .addFieldType(fieldType4)
  476.                 .addFieldType(fieldType5)
  477.                 .addFieldType(fieldType6)
  478.                 .addFieldType(fieldType7)
  479.                 .addFieldType(fieldType8)
  480.                 .build();
  481.         R<RpcStatus> response = milvusClient.createCollection(createCollectionReq);
  482.  
  483.         log.info("collection: {} is created ? status: = {}", collectionName, response.getData().getMsg());
  484.         return response.getData().getMsg().equals("Success");
  485.     }
  486.  
  487.  
  488.     public Boolean creatCollectionERPNLP(String collectionName) {
  489.         // 主键字段
  490.         FieldType fieldType1 = FieldType.newBuilder()
  491.                 .withName(Content.Field.ID)
  492.                 .withDescription("primary key")
  493.                 .withDataType(DataType.Int64)
  494.                 .withPrimaryKey(true)
  495.                 .withAutoID(true)
  496.                 .build();
  497.         // 文本字段
  498.         FieldType fieldType2 = FieldType.newBuilder()
  499.                 .withName(Content.Field.CONTENT)
  500.                 .withDataType(DataType.VarChar)
  501.                 .withMaxLength(Content.MAX_LENGTH)
  502.                 .build();
  503.         // 向量字段
  504.         FieldType fieldType3 = FieldType.newBuilder()
  505.                 .withName(Content.Field.CONTENT_VECTOR)
  506.                 .withDataType(DataType.FloatVector)
  507.                 .withDimension(Content.FEATURE_DIM_CLIP)
  508.                 .build();
  509.         FieldType fieldType4 = FieldType.newBuilder()
  510.                 .withName(Content.Field.CONTENT_ANSWER)
  511.                 .withDataType(DataType.VarChar)
  512.                 .withMaxLength(Content.MAX_LENGTH)
  513.                 .build();
  514.         FieldType fieldType5 = FieldType.newBuilder()
  515.                 .withName(Content.Field.TITLE)
  516.                 .withDataType(DataType.VarChar)
  517.                 .withMaxLength(Content.MAX_LENGTH)
  518.                 .build();
  519.         FieldType fieldType6 = FieldType.newBuilder()
  520.                 .withName(Content.Field.PARAM)
  521.                 .withDataType(DataType.VarChar)
  522.                 .withMaxLength(Content.MAX_LENGTH)
  523.                 .build();
  524.         FieldType fieldType7 = FieldType.newBuilder()
  525.                 .withName(Content.Field.TYPE)
  526.                 .withDataType(DataType.VarChar)
  527.                 .withMaxLength(Content.MAX_LENGTH)
  528.                 .build();
  529.         FieldType fieldType8 = FieldType.newBuilder()
  530.                 .withName(Content.Field.LABEL)
  531.                 .withDataType(DataType.VarChar)
  532.                 .withMaxLength(Content.MAX_LENGTH)
  533.                 .build();
  534.         // 创建collection
  535.         CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder()
  536.                 .withCollectionName(collectionName)
  537.                 .withDescription("Schema of Content ERP")
  538.                 .withShardsNum(Content.SHARDS_NUM)
  539.                 .addFieldType(fieldType1)
  540.                 .addFieldType(fieldType2)
  541.                 .addFieldType(fieldType3)
  542.                 .addFieldType(fieldType4)
  543.                 .addFieldType(fieldType5)
  544.                 .addFieldType(fieldType6)
  545.                 .addFieldType(fieldType7)
  546.                 .addFieldType(fieldType8)
  547.                 .build();
  548.         R<RpcStatus> response = milvusClient.createCollection(createCollectionReq);
  549.  
  550.         log.info("collection: {} is created ? status: = {}", collectionName, response.getData().getMsg());
  551.         return response.getData().getMsg().equals("Success");
  552.     }
  553.  
  554.     public List<List<SearchERPResultVo>> searchERPTopKSimilarity(SearchERPParamVo searchParamVo) {
  555.         log.info("------search ERP TopK Similarity------");
  556.         SearchParam searchParam = SearchParam.newBuilder()
  557.                 .withCollectionName(searchParamVo.getCollectionName())
  558.                 .withMetricType(MetricType.L2)
  559.                 .withOutFields(searchParamVo.getOutputFields())
  560.                 .withTopK(searchParamVo.getTopK())
  561.                 .withVectors(searchParamVo.getQueryVectors())
  562.                 .withVectorFieldName(Content.Field.CONTENT_VECTOR)
  563.                 .withParams(searchParamVo.getParams())
  564.                 .build();
  565.  
  566.         R<SearchResults> respSearch = milvusClient.search(searchParam);
  567.         if (respSearch.getData() == null) {
  568.             return null;
  569.         }
  570.         log.info("------ process query results ------");
  571.         SearchResultsWrapper wrapper = new SearchResultsWrapper(respSearch.getData().getResults());
  572.         List<List<SearchERPResultVo>> result = new ArrayList<>();
  573.         for (int i = 0; i < searchParamVo.getQueryVectors().size(); ++i) {
  574.             List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(i);
  575.             List<QueryResultsWrapper.RowRecord> rowRecords = wrapper.getRowRecords();
  576.             List<SearchERPResultVo> list = new ArrayList<>();
  577.             for (int j = 0; j < scores.size(); ++j) {
  578.                 SearchResultsWrapper.IDScore score = scores.get(j);
  579.                 QueryResultsWrapper.RowRecord rowRecord = rowRecords.get(j);
  580.                 long longID = score.getLongID();
  581.                 float distance = score.getScore();
  582.                 String content = (String) rowRecord.get(searchParamVo.getOutputFields().get(0));
  583.                 String contentAnswer = (String) rowRecord.get(searchParamVo.getOutputFields().get(1));
  584.                 String title = (String) rowRecord.get(searchParamVo.getOutputFields().get(2));
  585.                 log.info("Top " + j + " ID:" + longID + " Distance:" + distance);
  586.                 log.info("Content: " + content);
  587.                 list.add(SearchERPResultVo.builder().id(longID).score(distance).content(content).contentAnswer(contentAnswer).title(title).build());
  588.             }
  589.             result.add(list);
  590.         }
  591.         log.info("Successfully!");
  592.  
  593.         return result;
  594.     }
  595.  
  596.  
  597.     public List<List<SearchNLPResultVo>> searchNLPTopKSimilarity(SearchNLPParamVo searchParamVo) {
  598.         log.info("------search ERP TopK Similarity------");
  599.         SearchParam searchParam = SearchParam.newBuilder()
  600.                 .withCollectionName(searchParamVo.getCollectionName())
  601.                 .withMetricType(MetricType.L2)
  602.                 .withOutFields(searchParamVo.getOutputFields())
  603.                 .withTopK(searchParamVo.getTopK())
  604.                 .withVectors(searchParamVo.getQueryVectors())
  605.                 .withVectorFieldName(Content.Field.CONTENT_VECTOR)
  606.                 .withParams(searchParamVo.getParams())
  607.                 .withExpr(searchParamVo.getExpr())
  608.                 .build();
  609.  
  610.         R<SearchResults> respSearch = milvusClient.search(searchParam);
  611.         if (respSearch.getData() == null) {
  612.             return null;
  613.         }
  614.         log.info("------ process query results ------");
  615.         SearchResultsWrapper wrapper = new SearchResultsWrapper(respSearch.getData().getResults());
  616.         List<List<SearchNLPResultVo>> result = new ArrayList<>();
  617.         for (int i = 0; i < searchParamVo.getQueryVectors().size(); ++i) {
  618.             List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(i);
  619.             List<QueryResultsWrapper.RowRecord> rowRecords = wrapper.getRowRecords();
  620.             List<SearchNLPResultVo> list = new ArrayList<>();
  621.             for (int j = 0; j < scores.size(); ++j) {
  622.                 SearchResultsWrapper.IDScore score = scores.get(j);
  623.                 QueryResultsWrapper.RowRecord rowRecord = rowRecords.get(j);
  624.                 long longID = score.getLongID();
  625.                 float distance = score.getScore();
  626.                 String content = (String) rowRecord.get(searchParamVo.getOutputFields().get(0));
  627.                 String contentAnswer = (String) rowRecord.get(searchParamVo.getOutputFields().get(1));
  628.                 String title = (String) rowRecord.get(searchParamVo.getOutputFields().get(2));
  629.                 log.info("Top " + j + " ID:" + longID + " Distance:" + distance);
  630.                 log.info("Content: " + content);
  631.                 list.add(SearchNLPResultVo.builder().id(longID).score(distance).content(content).contentAnswer(contentAnswer).title(title).build());
  632.             }
  633.             result.add(list);
  634.         }
  635.         log.info("Successfully!");
  636.  
  637.         return result;
  638.     }
  639. }

3、测试用例 

MilvusServiceERPNLPTest

  1. @SpringBootTest(classes = {DataChatgptApplication.class}, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
  2. public class MilvusServiceERPNLPTest {
  3.  
  4.     @Autowired
  5.     MivusService milvusService;
  6.     @Autowired
  7.     MilvusClient milvusClient;
  8.  
  9.     @Test
  10.     void isExitCollection() {
  11.         boolean mediumArticles = milvusService.isExitCollection(Content.COLLECTION_NAME_NLP);
  12.         Assertions.assertTrue(mediumArticles);
  13.     }
  14.  
  15.  
  16.     @Test
  17.     void creatCollection() {
  18.         Boolean created = milvusService.creatCollectionERPNLP(Content.COLLECTION_NAME_NLP);
  19.         Assertions.assertTrue(created);
  20.     }
  21.  
  22.     @Test
  23.     void createIndex(){
  24.         Boolean index = milvusService.createIndex(Content.COLLECTION_NAME_NLP);
  25.         Assertions.assertTrue(index);
  26.     }
  27.  
  28.  
  29.     @Test
  30.     public void insertVector(){
  31.         List<String> sentenceList = new ArrayList<>();
  32.         sentenceList.add("网址是多少");
  33.       
  34.  
  35.  
  36.         List<String> contentAnswerList = new ArrayList<>();
  37.         contentAnswerList.add("/home.ashx");
  38.         
  39.  
  40.  
  41.         List<String> titleList = new ArrayList<>();
  42.         titleList.add("网址");
  43.        
  44.  
  45.  
  46.  
  47.         List<String> paramList = new ArrayList<>();
  48.         paramList.add("");
  49.         
  50.  
  51.  
  52.         List<String> typeList = new ArrayList<>();
  53.         typeList.add("0");
  54.         
  55.  
  56.  
  57.         List<String> labelList = new ArrayList<>();
  58.         labelList.add("操作直达");
  59.        
  60.  
  61.         PaddleNewTextVo paddleNewTextVo = null;
  62.         try {
  63.  
  64.             paddleNewTextVo = getVectorsLists(sentenceList);
  65.  
  66.             if (paddleNewTextVo == null) {
  67.                 // 获取不到再重试下
  68.                 paddleNewTextVo = getVectorsLists(sentenceList);
  69.             }
  70.             List<List<Double>> vectors = paddleNewTextVo.getVector();
  72.             List<List<Float>> floatVectors = new ArrayList<>();
  74.             for (List<Double> innerList : vectors) {
  75.                 List<Float> floatInnerList = new ArrayList<>();
  76.                 for (Double value : innerList) {
  77.                     floatInnerList.add(value.floatValue());
  78.                 }
  79.                 floatVectors.add(floatInnerList);
  80.             }
  81.  
  82.  
  83.             // 2.准备插入向量数据库
  84.             List<InsertParam.Field> fields = new ArrayList<>();
  85.             fields.add(new InsertParam.Field(Content.Field.CONTENT, sentenceList));
  86.             fields.add(new InsertParam.Field(Content.Field.CONTENT_VECTOR, floatVectors));
  87.             fields.add(new InsertParam.Field(Content.Field.CONTENT_ANSWER, contentAnswerList));
  88.             fields.add(new InsertParam.Field(Content.Field.TITLE, titleList));
  89.             fields.add(new InsertParam.Field(Content.Field.PARAM, paramList));
  90.             fields.add(new InsertParam.Field(Content.Field.TYPE, typeList));
  91.             fields.add(new InsertParam.Field(Content.Field.LABEL, labelList));
  92.  
  93.             // 3.执行操作
  94.             milvusService.insert(Content.COLLECTION_NAME_NLP, fields);
  95.  
  96.         } catch (ApiException e) {
  97.             System.out.println(e.getMessage());
  98.         } catch (IOException e) {
  99.             throw new RuntimeException(e);
  100.         }
  101.  
  102.     }
  103.  
  104.  
  105.  
  106.     private static PaddleNewTextVo getVectorsLists(List<String> sentenceList) throws IOException {
  107.  
  108.         String url = "http://192.168.1.243:6001/";  //paddle
  109.         URL obj = new URL(url);
  110.         HttpURLConnection con = (HttpURLConnection) obj.openConnection();
  111.  
  112.         // 设置超时时间
  113.         con.setConnectTimeout(50000);
  114.         con.setReadTimeout(200000);
  115.         con.setRequestMethod("POST");
  116.         con.setRequestProperty("Content-Type", "application/json");
  117.         con.setDoOutput(true);
  118.  
  119.         ObjectMapper objectParmMapper = new ObjectMapper();
  120.  
  121.         // 创建一个Map结构表示您的数据
  122.         Map<String, List<Map<String, String>>> dataMap = new HashMap<>();
  123.         dataMap.put("data", sentenceList.stream()
  124.                 .map(sentence -> Collections.singletonMap("text", sentence))
  125.                 .collect(Collectors.toList()));
  126.  
  127.         String jsonData = null;
  128.         try {
  129.             // 将Map转换为JSON字符串
  130.             jsonData = objectParmMapper.writeValueAsString(dataMap);
  131.         } catch (JsonProcessingException e) {
  132.             System.err.println("Error converting to JSON: " + e.getMessage());
  133.         }
  134.  
  135.  
  136.         String data = jsonData;
  137.  
  138.         try(OutputStream os = con.getOutputStream()) {
  139.             byte[] input = data.getBytes("utf-8");
  140.             os.write(input, 0, input.length);
  141.         }
  142.  
  143.         int responseCode = con.getResponseCode();
  144.         System.out.println("Response Code: " + responseCode);
  145.  
  146.         PaddleNewTextVo paddleNewTextVo = null;
  147.  
  148.         if (responseCode == HttpURLConnection.HTTP_OK) { // 200表示成功
  149.             BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
  150.             String inputLine;
  151.             StringBuilder content = new StringBuilder();
  152.  
  153.             while ((inputLine = in.readLine()) != null) {
  154.                 content.append(inputLine);
  155.             }
  156.  
  157.             in.close();
  158.  
  159.             try {
  160.                 String contentStr = content.toString();
  161.  
  162.                 // 直接解析JSON字符串到PaddleTextVo实例
  163.                 paddleNewTextVo = JSON.parseObject(contentStr, PaddleNewTextVo.class);
  164.  
  165.  
  166.             } catch (Exception e) {
  167.                 System.err.println("Error parsing JSON: " + e.getMessage());
  168.             }
  169.         } else {
  170.             System.out.println("Error Response Code: " + responseCode);
  171.             BufferedReader errorReader = new BufferedReader(new InputStreamReader(con.getErrorStream()));
  172.             String errorMessage;
  173.             while ((errorMessage = errorReader.readLine()) != null) {
  174.                 System.out.println("Error Message: " + errorMessage);
  175.             }
  176.             errorReader.close();
  177.         }
  178.         return paddleNewTextVo;
  179.     }
  180.  
  181.  
  182.  
  183.  
  184.  
  185.  
  186.     @Test
  187.     void searchTest(){
  188.         // 0.加载向量集合
  189.         milvusService.loadCollection(Content.COLLECTION_NAME_NLP);
  190.  
  191.         try {
  192.             List<String> sentenceList = new ArrayList<>();
  193.             sentenceList.add("XX列表");
  194.             String label = "操作直达";
  195.  
  196.             // 1.获得向量
  197.             // List<List<Float>> vectors = getVectorsLists(sentenceList);
  198.             List<List<Float>> vectors = new ArrayList<>();
  200.             SearchNLPParamVo searchParamVo = SearchNLPParamVo.builder()
  201.                     .collectionName(Content.COLLECTION_NAME_NLP)
  202.                     .queryVectors(vectors)
  203.                     .expr("label == '" + label + "'")
  204.                     .topK(3)
  205.                     .build();
  206.             // 2.在向量数据库中进行搜索内容知识
  207.             List<List<SearchNLPResultVo>> lists = milvusService.searchNLPTopKSimilarity(searchParamVo);
  208.             lists.forEach(searchResultVos -> {
  209.                 searchResultVos.forEach(searchResultVo -> {
  210.                     System.out.println(searchResultVo.getContent());
  211.                     System.out.println(searchResultVo.getContentAnswer());
  212.                     System.out.println(searchResultVo.getTitle());
  213.                     System.out.println(searchResultVo.getLabel());
  214.                 });
  215.             });
  216.  
  217.         } catch (ApiException e) {
  218.             System.out.println(e.getMessage());
  219.         } /*catch (IOException e) {
  220.             throw new RuntimeException(e);
  221.         }
  222. */
  223.     }
  224.  
  225.     @Test
  226.     public void insertTextVector() throws IOException {
  227.         List<String> titleList = new ArrayList<>();
  228.         List<String> sentenceList = new ArrayList<>();
  229.         List<String> contentAnswerList = new ArrayList<>();
  230.         List<String> paramList = new ArrayList<>();
  231.         List<String> typeList = new ArrayList<>();
  232.  
  233.  
  234.         String filePath = "src/main/resources/data/text.txt";
  235.  
  236.         try (BufferedReader reader = new BufferedReader(
  237.                 new InputStreamReader(new FileInputStream(filePath), StandardCharsets.UTF_8))) {
  238.             // 使用4个竖线（||||）作为分隔符
  239.             String line;
  240.             while ((line = reader.readLine()) != null) {
  241.                 String[] parts = line.split("\\|\\|\\|\\|");
  242.  
  243.                 if (parts.length >= 3) {
  244.                     titleList.add(parts[0].trim());
  245.                     sentenceList.add(parts[1].trim());
  246.                     contentAnswerList.add(parts[2].trim());
  247.                     paramList.add("");
  248.                     typeList.add("2");
  249.                 } else {
  250.                     System.out.println("Warning: Invalid format on line: " + line);
  251.                 }
  252.             }
  253.  
  254.             // 打印或处理列表内容
  255.             System.out.println("Title List: " + titleList);
  256.             System.out.println("Sentence List: " + sentenceList);
  257.             System.out.println("Content Answer List: " + contentAnswerList);
  258.  
  259.         } catch (IOException e) {
  260.             System.err.println("Error reading file: " + e.getMessage());
  261.         }
  262.  
  263.  
  264.         try {
  265.             // 1.获得向量
  266.             TextEmbeddingParam param = TextEmbeddingParam
  267.                     .builder()
  268.                     .model(TextEmbedding.Models.TEXT_EMBEDDING_V1)
  269.                     .texts(sentenceList).build();
  270.             TextEmbedding textEmbedding = new TextEmbedding();
  271.             TextEmbeddingResult result = textEmbedding.call(param);
  272.  
  273.             List<List<Float>> vectors = new ArrayList<>();
  275.             for (int i = 0; i < result.getOutput().getEmbeddings().size(); i++) {
  276.                 List<Double> vector = result.getOutput().getEmbeddings().get(i).getEmbedding();
  277.                 List<Float> floatVector = vector.stream()
  278.                         .map(Double::floatValue)
  279.                         .collect(Collectors.toList());
  280.                 vectors.add(floatVector);
  281.             }
  282.  
  283.  
  284.             // 2.准备插入向量数据库
  285.             List<InsertParam.Field> fields = new ArrayList<>();
  286.             fields.add(new InsertParam.Field(Content.Field.CONTENT, sentenceList));
  287.             fields.add(new InsertParam.Field(Content.Field.CONTENT_VECTOR, vectors));
  288.             fields.add(new InsertParam.Field(Content.Field.CONTENT_ANSWER, contentAnswerList));
  289.             fields.add(new InsertParam.Field(Content.Field.TITLE, titleList));
  290.             fields.add(new InsertParam.Field(Content.Field.PARAM, paramList));
  291.             fields.add(new InsertParam.Field(Content.Field.TYPE, typeList));
  292.             // 3.执行操作
  293.             milvusService.insert(Content.COLLECTION_NAME_NLP, fields);
  294.  
  295.         } catch (ApiException | NoApiKeyException e) {
  296.             System.out.println(e.getMessage());
  297.         }
  298.     }
  299.  
  300.     @Test
  301.     void ChatBasedContentTest() throws NoApiKeyException, InputRequiredException, InterruptedException {
  302.         // 0.加载向量集合
  303.         milvusService.loadCollection(Content.COLLECTION_NAME_NLP);
  304.  
  305.         try {
  306.             String question = "查询订单";
  307.             List<String> sentenceList = new ArrayList<>();
  308.             sentenceList.add(question);
  309.  
  310.             // 1.获得向量
  311.             TextEmbeddingParam param = TextEmbeddingParam
  312.                     .builder()
  313.                     .model(TextEmbedding.Models.TEXT_EMBEDDING_V1)
  314.                     .texts(sentenceList).build();
  315.             TextEmbedding textEmbedding = new TextEmbedding();
  316.             TextEmbeddingResult result = textEmbedding.call(param);
  317.  
  318.             List<Double> vector = result.getOutput().getEmbeddings().get(0).getEmbedding();
  319.  
  320.             List<Float> floatVector = vector.stream()
  321.                     .map(Double::floatValue)
  322.                     .collect(Collectors.toList());
  323.             List<List<Float>> vectors = Collections.singletonList(floatVector);
  325.             SearchERPParamVo searchParamVo = SearchERPParamVo.builder()
  326.                     .collectionName(Content.COLLECTION_NAME_NLP)
  327.                     .queryVectors(vectors)
  328.                     .topK(3)
  329.                     .build();
  330.             // 2.在向量数据库中进行搜索内容知识
  331.             StringBuffer buffer = new StringBuffer();
  332.             List<List<SearchERPResultVo>> lists = milvusService.searchERPTopKSimilarity(searchParamVo);
  333.             lists.forEach(searchResultVos -> {
  334.                 searchResultVos.forEach(searchResultVo -> {
  335.                     buffer.append("问题: " + searchResultVo.getContent());
  336.                     buffer.append("答案: " + searchResultVo.getContentAnswer());
  337.                 });
  338.             });
  339.             // 3.进行对话
  340.             String prompt = "请你充分理解下面的内容，然后回答问题, 要求仅返回答案[]中内容：";
  341.             String content = buffer.toString();
  342.             String resultQwen = streamCallWithCallback(prompt + content + question);
  343.             // System.out.println(resultQwen);
  344.  
  345.         } catch (ApiException | NoApiKeyException e) {
  346.             System.out.println(e.getMessage());
  347.         }
  348.  
  349.     }
  350.  
  351.  
  352.     public static String streamCallWithCallback(String content)
  353.             throws NoApiKeyException, ApiException, InputRequiredException,InterruptedException {
  354.         Constants.apiKey="sk-2106098eed1f43c9bde754f3e87038a2";
  355.         Generation gen = new Generation();
  356.         Message userMsg = Message
  357.                 .builder()
  358.                 .role(Role.USER.getValue())
  359.                 .content(content)
  360.                 .build();
  361.         QwenParam param = QwenParam
  362.                 .builder()
  363.                 .model(Generation.Models.QWEN_PLUS)
  364.                 .resultFormat(QwenParam.ResultFormat.MESSAGE)
  365.                 .messages(Arrays.asList(userMsg))
  366.                 .topP(0.8)
  367.                 .incrementalOutput(true) // get streaming output incrementally
  368.                 .build();
  369.         Semaphore semaphore = new Semaphore(0);
  370.         StringBuilder fullContent = new StringBuilder();
  371.         gen.streamCall(param, new ResultCallback<GenerationResult>() {
  372.  
  373.             @Override
  374.             public void onEvent(GenerationResult message) {
  375.                 fullContent.append(message.getOutput().getChoices().get(0).getMessage().getContent());
  376.                 System.out.println(message);
  377.             }
  378.             @Override
  379.             public void onError(Exception err){
  380.                 System.out.println(String.format("Exception: %s", err.getMessage()));
  381.                 semaphore.release();
  382.             }
  383.  
  384.             @Override
  385.             public void onComplete(){
  386.                 System.out.println("Completed");
  387.                 semaphore.release();
  388.             }
  389.  
  390.         });
  391.         semaphore.acquire();
  392.         System.out.println("Full content: \n" + fullContent.toString());
  393.         return fullContent.toString();
  394.     }
  395.  
  396.     @Test
  397.     void loadData() throws IOException {
  398.         // Read the dataset file
  399.         String content = readFileToString("src/main/resources/data/medium_articles_2020_dpr.json");
  400.         // Load dataset
  401.         JSONObject dataset = JSON.parseObject(content);
  402.         List<JSONObject> rows = getRows(dataset.getJSONArray("rows"), 2);
  403.         System.out.println(rows);
  404.     }
  405.  
  406.     public String readFileToString(String filePath) throws IOException {
  407.         return new String(Files.readAllBytes(Paths.get(filePath)), StandardCharsets.UTF_8);
  408.     }
  409.     public static List<JSONObject> getRows(JSONArray dataset, int counts) {
  410.         List<JSONObject> rows = new ArrayList<>();
  411.         for (int i = 0; i < counts; i++) {
  412.             JSONObject row = dataset.getJSONObject(i);
  413.             List<Float> vectors = row.getJSONArray("title_vector").toJavaList(Float.class);
  414.             Long reading_time = row.getLong("reading_time");
  415.             Long claps = row.getLong("claps");
  416.             Long responses = row.getLong("responses");
  417.             row.put("title_vector", vectors);
  418.             row.put("reading_time", reading_time);
  419.             row.put("claps", claps);
  420.             row.put("responses", responses);
  421.             row.remove("id");
  422.             rows.add(row);
  423.         }
  424.         return rows;
  425.     }
  426.  
  427.     @Test
  428.     void getFileds() throws IOException {
  429.         String content = readFileToString("src/main/resources/data/medium_articles_2020_dpr.json");
  430.  
  431.         // Load dataset
  432.         JSONObject dataset = JSON.parseObject(content);
  433.         List<InsertParam.Field> field = getFields(dataset.getJSONArray("rows"), 1);
  434.         System.out.println(field);
  435.     }
  436.  
  437.     public static List<InsertParam.Field> getFields(JSONArray dataset, int counts) {
  438.         List<InsertParam.Field> fields = new ArrayList<>();
  439.         List<String> titles = new ArrayList<>();
  440.         List<List<Float>> title_vectors = new ArrayList<>();
  441.         List<String> links = new ArrayList<>();
  442.         List<Long> reading_times = new ArrayList<>();
  443.         List<String> publications = new ArrayList<>();
  444.         List<Long> claps_list = new ArrayList<>();
  445.         List<Long> responses_list = new ArrayList<>();
  446.  
  447.         for (int i = 0; i < counts; i++) {
  448.             JSONObject row = dataset.getJSONObject(i);
  449.             titles.add(row.getString("title"));
  450.             title_vectors.add(row.getJSONArray("title_vector").toJavaList(Float.class));
  451.             links.add(row.getString("link"));
  452.             reading_times.add(row.getLong("reading_time"));
  453.             publications.add(row.getString("publication"));
  454.             claps_list.add(row.getLong("claps"));
  455.             responses_list.add(row.getLong("responses"));
  456.         }
  457.  
  458.         fields.add(new InsertParam.Field("title", titles));
  459.         fields.add(new InsertParam.Field("title_vector", title_vectors));
  460.         fields.add(new InsertParam.Field("link", links));
  461.         fields.add(new InsertParam.Field("reading_time", reading_times));
  462.         fields.add(new InsertParam.Field("publication", publications));
  463.         fields.add(new InsertParam.Field("claps", claps_list));
  464.         fields.add(new InsertParam.Field("responses", responses_list));
  465.  
  466.         return fields;
  467.     }
  468.  
  469.  
  470.     @Test
  471.     void searchTopKSimilarity() throws IOException {
  472.         // Search data
  473.         String content = readFileToString("src/main/resources/data/medium_articles_2020_dpr.json");
  474.         // Load dataset
  475.         JSONObject dataset = JSON.parseObject(content);
  476.         List<JSONObject> rows = getRows(dataset.getJSONArray("rows"), 10);
  477.         // You should include the following in the main function
  478.  
  479.         List<List<Float>> queryVectors = new ArrayList<>();
  480.         List<Float> queryVector = rows.get(0).getJSONArray("title_vector").toJavaList(Float.class);
  481.         queryVectors.add(queryVector);
  482.  
  483.         // Prepare the outputFields
  484.         List<String> outputFields = new ArrayList<>();
  485.         outputFields.add("title");
  486.         outputFields.add("link");
  487.  
  488.         // Search vectors in a collection
  489.         SearchParam searchParam = SearchParam.newBuilder()
  490.                 .withCollectionName("medium_articles")
  491.                 .withVectorFieldName("title_vector")
  492.                 .withVectors(queryVectors)
  493.                 .withExpr("claps > 30 and reading_time < 10")
  494.                 .withTopK(3)
  495.                 .withMetricType(MetricType.L2)
  496.                 .withParams("{\"nprobe\":10,\"offset\":2, \"limit\":3}")
  497.                 .withConsistencyLevel(ConsistencyLevelEnum.BOUNDED)
  498.                 .withOutFields(outputFields)
  499.                 .build();
  500.  
  501.         R<SearchResults> response = milvusClient.search(searchParam);
  502.  
  503.         SearchResultsWrapper wrapper = new SearchResultsWrapper(response.getData().getResults());
  504.         System.out.println("Search results");
  505.         for (int i = 0; i < queryVectors.size(); ++i) {
  506.             List<SearchResultsWrapper.IDScore> scores = wrapper.getIDScore(i);
  507.             List<QueryResultsWrapper.RowRecord> rowRecords = wrapper.getRowRecords();
  508.             for (int j = 0; j < scores.size(); ++j) {
  509.                 SearchResultsWrapper.IDScore score = scores.get(j);
  510.                 QueryResultsWrapper.RowRecord rowRecord = rowRecords.get(j);
  511.                 System.out.println("Top " + j + " ID:" + score.getLongID() + " Distance:" + score.getScore());
  512.                 System.out.println("Title: " + rowRecord.get("title"));
  513.                 System.out.println("Link: " + rowRecord.get("link"));
  514.             }
  515.         }
  516.  
  517.     }
  518.  
  519.    
  520.     
  521. }

4、查询

// 先根据向量查询语义相近的语料
List<Question> questionList = mivusService.searchNewPaddleQuestion(req.getMessage(), "1", appType);

  1. **
  2.      * 根据问题进行向量查询,采用Paddle服务 采用新的文本分类方法
  3.      * @param question 用户的问题文本
  4.      * @return 相关的初始问题知识列表
  5.      */
  6.     public List<Question> searchNewPaddleQuestion(String question, String type, String appType) {
  7.  
  8.         // 0.加载向量集合
  9.         String collection = Content.COLLECTION_NAME_NLP;
  10.         if (appType.equals("1")) {
  11.             collection = Content.COLLECTION_NAME_NLP_APP;
  12.         }
  13.         loadCollection(collection);
  14.  
  15.         List<Question> resultList = new LinkedList<>();
  16.  
  17.         PaddleNewTextVo paddleNewTextVo = null;
  18.         try {
  19.             List<String> sentenceList = new ArrayList<>();
  20.             sentenceList.add(question);
  21.  
  22.             // 1.获得向量
  23.             paddleNewTextVo = getNewNLPVectorsLists(sentenceList);
  24.             log.info("实时向量值 : {}", paddleNewTextVo.getPredictedList());
  25.  
  26.             List<List<Double>> vectors = paddleNewTextVo.getVector();
  28.             List<List<Float>> floatVectors = new ArrayList<>();
  30.             for (List<Double> innerList : vectors) {
  31.                 List<Float> floatInnerList = new ArrayList<>();
  32.                 for (Double value : innerList) {
  33.                     floatInnerList.add(value.floatValue());
  34.                 }
  35.                 floatVectors.add(floatInnerList);
  36.             }
  37.  
  38.             List<Integer> predictedList = paddleNewTextVo.getPredictedList();
  39.  
  40.             List<String> labelStrings = new ArrayList<>();
  41.             HashSet<Integer> setType = new HashSet();
  42.             int topK = 3;
  43.             if(!predictedList.isEmpty()) {
  44.                 // 去重
  45.                 for (Integer number : predictedList) {
  46.                     setType.add(number);
  47.                     if (number == 2) {
  48.                         // 如何是 2
  49.                         topK = 1;
  50.                     }
  51.                 }
  52.                 for (Integer label : setType) {
  53.                     labelStrings.add("'" + label + "'");
  54.                 }
  55.             }
  56.             String typeResult = "[" + String.join(", ", labelStrings) + "]";
  57.  
  58.             SearchNLPParamVo searchParamVo = SearchNLPParamVo.builder()
  59.                     .collectionName(collection)
  60.                     //.expr("type == '" + type + "'")
  61.                     .expr("type in ['0','1','2']")
  62.                     //.expr("type in " + typeResult + " ")
  63.                     .queryVectors(floatVectors)
  64.                     .topK(topK)
  65.                     .build();
  66.             // 2.在向量数据库中进行搜索内容知识
  67.             List<List<SearchNLPResultVo>> lists = searchNLPERPTopKSimilarity(searchParamVo);
  68.             lists.forEach(searchResultVos -> {
  69.                 searchResultVos.forEach(searchResultVo -> {
  70.                     log.info(searchResultVo.getContent());
  71.                     log.info(searchResultVo.getContentAnswer());
  72.                     Question question1 = new Question();
  73.                     question1.setQuestionId(Long.valueOf(searchResultVo.getId()));
  74.                     question1.setQuestion(searchResultVo.getContent());
  75.                     question1.setAnswer(searchResultVo.getContentAnswer());
  76.                     question1.setTitle(searchResultVo.getTitle());
  77.                     question1.setParam(searchResultVo.getParam());
  78.                     question1.setType(searchResultVo.getType());
  79.                     question1.setLabel(searchResultVo.getLabel());
  80.                     resultList.add(question1);
  81.                 });
  82.             });
  83.  
  84.         } catch (ApiException | IOException e) {
  85.             log.error(e.getMessage());
  86.         }
  87.  
  88.         // 将查询到的结果转换为之前构造的 Question 的格式返回给前端
  89.         return resultList;
  90.     }

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/代码探险家/article/detail/823402
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号