devbox
秋刀鱼在做梦
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

vLLM部署qwen大模型加速推理实现_vllm 启动qwen

作者：秋刀鱼在做梦 | 2024-07-26 13:13:39

赞

踩

vllm 启动qwen

目录

step1: 编写 vllm_wrapper.py

step2: 应用场景：给体检指标生成健康建议

step1: 编写 vllm_wrapper.py

  1. from transformers import PreTrainedTokenizer, GenerationConfig, StoppingCriteriaList
  2. from typing import Optional, Callable, List, Tuple, Union
  3. import copy
  4. import torch
  5. from transformers import AutoTokenizer
  6. from transformers.generation.logits_process import LogitsProcessorList
  7. from packaging import version
  8.  
  9. _ERROR_BAD_CHAT_FORMAT = """\
  10. We detect you are probably using the pretrained model (rather than chat model) for chatting, since the chat_format in generation_config is not "chatml".
  11. If you are directly using the model downloaded from Huggingface, please make sure you are using our "Qwen/Qwen-7B-Chat" Huggingface model (rather than "Qwen/Qwen-7B") when you call model.chat().
  12. 我们检测到您可能在使用预训练模型（而非chat模型）进行多轮chat，因为您当前在generation_config指定的chat_format，并未设置为我们在对话中所支持的"chatml"格式。
  13. 如果您在直接使用我们从Huggingface提供的模型，请确保您在调用model.chat()时，使用的是"Qwen/Qwen-7B-Chat"模型（而非"Qwen/Qwen-7B"预训练模型）。
  14. """
  15.  
  16. IMEND = "<|im_end|>"
  17. ENDOFTEXT = "<|endoftext|>"
  18.  
  19. HistoryType = List[Tuple[str, str]]
  20. TokensType = List[int]
  21. BatchTokensType = List[List[int]]
  22.  
  23. def get_stop_words_ids(chat_format, tokenizer):
  24.     if chat_format == "raw":
  25.         stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]]
  26.     elif chat_format == "chatml":
  27.         stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]]
  28.     else:
  29.         raise NotImplementedError(f"Unknown chat format {chat_format!r}")
  30.     return stop_words_ids
  31.  
  32. def make_context(
  33.     tokenizer: PreTrainedTokenizer,
  34.     query: str,
  35.     history: List[Tuple[str, str]] = None,
  36.     system: str = "",
  37.     max_window_size: int = 6144,
  38.     chat_format: str = "chatml",
  39. ):
  40.     if history is None:
  41.         history = []
  42.  
  43.     if chat_format == "chatml":
  44.         im_start, im_end = "<|im_start|>", "<|im_end|>"
  45.         im_start_tokens = [tokenizer.im_start_id]
  46.         im_end_tokens = [tokenizer.im_end_id]
  47.         nl_tokens = tokenizer.encode("\n")
  48.  
  49.         def _tokenize_str(role, content):
  50.             return f"{role}\n{content}", tokenizer.encode(
  51.                 role, allowed_special=set()
  52.             ) + nl_tokens + tokenizer.encode(content, allowed_special=set())
  53.  
  54.         system_text, system_tokens_part = _tokenize_str("system", system)
  55.         system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
  56.  
  57.         raw_text = ""
  58.         context_tokens = []
  59.  
  60.         for turn_query, turn_response in reversed(history):
  61.             query_text, query_tokens_part = _tokenize_str("user", turn_query)
  62.             query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
  63.             response_text, response_tokens_part = _tokenize_str(
  64.                 "assistant", turn_response
  65.             )
  66.             response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
  67.  
  68.             next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
  69.             prev_chat = (
  70.                 f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
  71.             )
  72.  
  73.             current_context_size = (
  74.                 len(system_tokens) + len(next_context_tokens) + len(context_tokens)
  75.             )
  76.             if current_context_size < max_window_size:
  77.                 context_tokens = next_context_tokens + context_tokens
  78.                 raw_text = prev_chat + raw_text
  79.             else:
  80.                 break
  81.  
  82.         context_tokens = system_tokens + context_tokens
  83.         raw_text = f"{im_start}{system_text}{im_end}" + raw_text
  84.         context_tokens += (
  85.             nl_tokens
  86.             + im_start_tokens
  87.             + _tokenize_str("user", query)[1]
  88.             + im_end_tokens
  89.             + nl_tokens
  90.             + im_start_tokens
  91.             + tokenizer.encode("assistant")
  92.             + nl_tokens
  93.         )
  94.         raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
  95.  
  96.     elif chat_format == "raw":
  97.         raw_text = query
  98.         context_tokens = tokenizer.encode(raw_text)
  99.     else:
  100.         raise NotImplementedError(f"Unknown chat format {chat_format!r}")
  101.  
  102.     return raw_text, context_tokens
  103.  
  104. class vLLMWrapper:
  105.     def __init__(self,
  106.                model_dir: str,
  107.                trust_remote_code: bool = True,
  108.                tensor_parallel_size: int = 1,
  109.                gpu_memory_utilization: float = 0.98,
  110.                dtype: str = "bfloat16",
  111.                **kwargs):
  112.  
  113.         if dtype not in ("bfloat16", "float16", "float32"):
  114.             print("now not support {}!".format(dtype))
  115.             raise Exception
  116.  
  117.         # build generation_config
  118.         self.generation_config = GenerationConfig.from_pretrained(model_dir, trust_remote_code=trust_remote_code)
  119.  
  120.         # build tokenizer
  121.         self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
  122.         self.tokenizer.eos_token_id = self.generation_config.eos_token_id
  123.  
  124.         self.stop_words_ids = []
  125.  
  126.         from vllm import LLM
  127.         import vllm
  128.         if version.parse(vllm.__version__) >= version.parse("0.2.2"):
  129.             self.__vllm_support_repetition_penalty = True
  130.         else:
  131.             self.__vllm_support_repetition_penalty = False
  132.  
  133.         quantization = getattr(kwargs, 'quantization', None)
  134.  
  135.         self.model = LLM(model=model_dir,
  136.                             tokenizer=model_dir,
  137.                             tensor_parallel_size=tensor_parallel_size,
  138.                             trust_remote_code=trust_remote_code,
  139.                             quantization=quantization,
  140.                             gpu_memory_utilization=gpu_memory_utilization,
  141.                             dtype=dtype)
  142.  
  143.         for stop_id in get_stop_words_ids(self.generation_config.chat_format, self.tokenizer):
  144.             self.stop_words_ids.extend(stop_id)
  145.         self.stop_words_ids.extend([self.generation_config.eos_token_id])
  146.  
  147.     def chat(self,
  148.         query: str,
  149.         history: Optional[HistoryType],
  150.         tokenizer: PreTrainedTokenizer = None,
  151.         system: str = "You are a helpful assistant.",
  152.         generation_config: Optional[GenerationConfig] = None,
  153.         **kwargs):
  154.         generation_config = generation_config if generation_config is not None else self.generation_config
  155.         tokenizer = self.tokenizer if tokenizer is None else tokenizer
  156.  
  157.         assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
  158.         if not self.__vllm_support_repetition_penalty and generation_config.repetition_penalty != 1:
  159.             raise RuntimeError("The installed vLLM doesn't support repetition_penalty, please set ``model.generation_config.repetition_penalty = 1`` or install vllm>=0.2.2")
  160.  
  161.         if history is None:
  162.             history = []
  163.         else:
  164.             # make a copy of the user's input such that is is left untouched
  165.             history = copy.deepcopy(history)
  166.  
  167.         extra_stop_words_ids = kwargs.get('stop_words_ids', None)
  168.         if extra_stop_words_ids is None:
  169.             extra_stop_words_ids = []
  170.  
  171.         max_window_size = kwargs.get('max_window_size', None)
  172.         if max_window_size is None:
  173.             max_window_size = generation_config.max_window_size
  174.  
  175.         from vllm.sampling_params import SamplingParams
  176.         sampling_kwargs = {
  177.             "stop_token_ids": self.stop_words_ids,
  178.             "early_stopping": False,
  179.             "top_p": generation_config.top_p,
  180.             "top_k": -1 if generation_config.top_k == 0 else generation_config.top_k,
  181.             "temperature": generation_config.temperature,
  182.             "max_tokens": generation_config.max_new_tokens,
  183.             "repetition_penalty": generation_config.repetition_penalty
  184.         }
  185.         if not self.__vllm_support_repetition_penalty:
  186.             sampling_kwargs.pop("repetition_penalty")
  187.         sampling_params = SamplingParams(**sampling_kwargs)
  188.  
  189.         raw_text, context_tokens = make_context(
  190.             self.tokenizer,
  191.             query,
  192.             history=history,
  193.             system=system,
  194.             max_window_size=max_window_size,
  195.             chat_format=generation_config.chat_format,
  196.         )
  197.  
  198.         req_outputs = self.model.generate([query],
  199.                                             sampling_params=sampling_params,
  200.                                             prompt_token_ids=[context_tokens])
  201.         req_output = req_outputs[0]
  202.  
  203.         prompt_str = req_output.prompt
  204.         prompt_ids = req_output.prompt_token_ids
  205.         req_sample_output_ids = []
  206.         req_sample_output_strs = []
  207.         for sample in req_output.outputs:
  208.             output_str = sample.text
  209.             output_ids = sample.token_ids
  210.             if IMEND in output_str:
  211.                 output_str = output_str[:-len(IMEND)]
  212.             if ENDOFTEXT in output_str:
  213.                 output_str = output_str[:-len(ENDOFTEXT)]
  214.             req_sample_output_ids.append(prompt_ids + output_ids)
  215.             req_sample_output_strs.append(prompt_str + output_str)
  216.         assert len(req_sample_output_strs) == 1
  217.         response = req_sample_output_strs[0][len(prompt_str):]
  218.         history.append((prompt_str, response))
  219.  
  220.         return response, history
  221.  
  222. if __name__ == '__main__':
  223.  
  224.     model_dir = 'Qwen/Qwen-72B-Chat'
  225.     tensor_parallel_size = 2
  226.  
  227.     model = vLLMWrapper(model_dir,
  228.                         tensor_parallel_size=tensor_parallel_size,
  229.                         )
  230.  
  231.     response, history = model.chat(query="你好",
  232.                                    history=None)
  233.     print(response)
  234.     response, history = model.chat(query="给我讲一个年轻人奋斗创业最终取得成功的故事。",
  235.                                    history=history)
  236.     print(response)
  237.     response, history = model.chat(query="给这个故事起一个标题",
  238.                                    history=history)
  239.     print(response)

step2: 应用场景：给体检指标生成健康建议

  1.  
  2. import pandas as pd
  3. from peft import AutoPeftModelForCausalLM
  4. from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig
  5. from vllm_wrapper import vLLMWrapper
  6. from tqdm import tqdm
  7.  
  8. sft_path = "/mnt/sdd/Qwen-7B-Chat"
  9. tokenizer = AutoTokenizer.from_pretrained(sft_path, trust_remote_code=True)
  10. model = vLLMWrapper(sft_path, tensor_parallel_size=1)
  11. # model = AutoModelForCausalLM.from_pretrained(sft_path, device_map="auto", trust_remote_code=True).eval()
  12.  
  13. import uuid
  14.  
  15. def data_load():
  16.     zb = pd.read_csv('/home/wangyp/Big_Model/infectious_disease/data/zb.csv', header=0)
  17.     # print(zb.head(10))
  18.     nums = []
  19.     yins = {}
  20.     nulls = []
  21.     
  22.     for item in zb.itertuples(index=False):
  23.         if item.normal_desc is not None and item.normal_desc.strip() != '':
  24.             yins[item.NAME.strip()] = item.normal_desc.strip()
  25.         if( (item.range_floor is not None and str(item.range_floor).strip() != '') and (item.range_ceil is not None and str(item.range_ceil).strip() != '') ):
  26.             nums.append({item.NAME.strip(): str(item.range_floor).strip() + "-" + str(item.range_ceil).strip() })
  27.         else:
  28.             nulls.append(item.NAME.strip())
  29.     return yins, nums, nulls
  30.  
  31. def zao_yins(yins):
  32.     res = []
  33.     for key in yins.keys():
  34.         res.append( key + "这种体检指标阳性。")
  35.         res.append( key + "的检测结果为阳性。")
  36.         res.append( key + "的检测显示出阳性反应。")
  37.         res.append( key + "检测结果为阳性。")
  38.         res.append( key + "的阳性结果在体检中被检测到。")
  39.         res.append( key + "试验为阳性状态")
  40.         res.append( key + "的阳性结果在体检中得到了确认。")
  41.         res.append( key + "的检测结果表明为阳性。")
  42.         res.append( "在进行检测" + key + "指标时，结果被判定为阳性。")
  43.     return res
  44.  
  45. def zao_nums(nums):
  46.     res = []
  47.     keys_list = [key for d in nums for key in d.keys()]
  48.     for key in keys_list:
  49.         res.append({"name": key, "value": key + "检测结果显示异常。"})
  50.         # res.append({"name": key, "value": key + "的检查值超出了正常范围。"})
  51.         res.append({"name": key, "value": key + "的测量值与标准值不符。"})
  52.         # res.append({"name": key, "value": key + "检测结果呈现异常状态。"})
  53.         res.append({"name": key, "value": key + "的数值在体检中被标记为异常。"})
  54.         # res.append({"name": key, "value": key + "检查结果显示了不正常的数值。"})
  55.         res.append({"name": key, "value": key + "的检测结果不在正常参考值内。"})
  56.         # res.append({"name": key, "value": key + "检查报告提示数值异常。"})
  57.         # res.append({"name": key, "value": "体检报告指出" + key + "水平不正常。"})
  58.         res.append({"name": key, "value": "体检中发现" + key + "水平异常。"})
  59.         # res.append(key + "检测结果显示异常。")
  60.  
  61.     return res
  62. # 体检中发现尿酮水平异常，帮我生成10条描述，保持句子意思不变
  63. def z_nulls(nulls):
  64.     res = []
  65.     for key in nulls:
  66.         res.append("体检结果显示" + key + "水平出现异常。")
  67.         res.append("在进行体检时，发现" + key + "的数值不在正常范围内。")
  68.         res.append("体检报告中指出" + key + "水平有异常情况。")
  69.         res.append("体检时，" + key + "水平的测定结果超出了预期的正常值。")
  70.         res.append("体检中测得的" + key + "水平与正常值有所偏差。")
  71.         res.append("体检数据中，" + key + "的数值检测出异常。")
  72.         res.append(key + "的检测结果表明存在异常。")
  73.         res.append(key + "的检测值在体检中被标记为异常。")
  74.         res.append(key + "水平的体检结果提示有异常。")
  75.     return res
  76.  
  77. yins_template = """
  79. """
  80.  
  81. # 定义一个带有槽位的字符串模板
  82. # yins_template = "Hello, {name}! You are {age} years old."
  83. yins_template = """hhh******************"""
  84.  
  85. nums_template = """
  86. {disc}你是一名体检报告领域的专家，请生成一段关于该体检指标异常的改善建议。\n下面是生成体检指标相关的建议时的要求：健康建议严格包含如下几项：复检确认、营养评估、医疗咨询、健康饮食、生活方式调整、药物治疗、定期监测、记录症状这几项。生成建议请参考以下格式：\n体检结果提示您的{name}不在正常参考值内，这可能与多种因素有关，包括营养不良、维生素缺乏或某些疾病状态。以下是一些建议：\n复检确认：{name}相关的复检建议。\n营养评估：考虑针对{name}进行一次全面的营养评估。\n医疗咨询：咨询医生，以确定是否需要进一步的检查和{name}相关的其他检测。如血红蛋白电泳、血清铁蛋白、维生素B12和叶酸水平检测。\n健康饮食：饮食建议，这些食物富含补充{name}必要的营养素。\n生活方式调整：保持适度的体育活动，避免饮酒和吸烟，这些都可能影响{name}的健康。\n药物治疗：如果医生建议，可能需要服用补充剂或药物来纠正{name}异常。\n定期监测：根据医生的建议，定期监测{name}和其他{name}相关指标。\n记录症状：注意任何可能与{name}相关的症状，如疲劳、头晕或呼吸困难，并及时告知医生。\n请记住，{name}的异常可能是多种情况的指标，因此重要的是遵循医疗专业人员的指导进行进一步的评估和治疗。\n
  87. """
  88. # filled_nums_template = nums_template.format(name=name)
  89.  
  90.  
  91. def load_model():
  92.     pass
  93.     
  94. if __name__ == '__main__':
  95.     all = []
  96.     yins, nums, nulls = data_load()
  97.     # 遍历，一个字段造10个template，存储到list中，写入文件
  98.     yins_tem = zao_yins(yins)
  99.     nums_tem = zao_nums(nums)
  100.     nulls_tem = z_nulls(nulls)
  101.     # all = yins_tem + nums_tem + nulls_tem
  102.     # print(len(all))
  103.     nums_conversations = []
  104.     for num in tqdm(nums_tem):
  105.         filled_nums_template = nums_template.format(disc=num["value"], name=num["name"])
  106.         response, history = model.chat(filled_nums_template, history=None)
  107.         nums_conversations.append({"id": str(uuid.uuid4()), "conversations": [{"from": "user", "value": num["value"]}, {"from": "assistant", "value": response}]})
  108.  
  109.     with open("/home/wangyp/Big_Model/infectious_disease/data/zb_train.json", "w", encoding="utf-8") as f:
  110.         f.write(",\n".join(str(i) for i in nums_conversations))
  111.  
  112.     print("nums_conversations数据处理完毕。。。。。。。。。。。。。。。。。。。")

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/秋刀鱼在做梦/article/detail/885491
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号