devbox
很楠不爱3
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

模型运行页面设计webui代码_torch_gc

作者：很楠不爱3 | 2024-04-11 14:16:56

赞

踩

torch_gc
  1. from deep_training.data_helper import ModelArguments, TrainingArguments, DataArguments
  2. from deep_training.nlp.models.chatglm import setup_model_profile, ChatGLMConfig
  3. from deep_training.nlp.models.lora.v2 import LoraArguments
  4. from transformers import HfArgumentParser
  5. from typing import Optional, List, Tuple
  6. from data_utils import train_info_args, NN_DataHelper
  7. from models import MyTransformer,ChatGLMTokenizer
  8.  
  9. import os
  10.  
  11. import gradio as gr
  12.  
  13. from webui.context import ctx
  14. from webui.device import torch_gc
  15. css = "style.css"
  16. script_path = "scripts"
  17. _gradio_template_response_orig = gr.routes.templates.TemplateResponse
  18.  
  19. # 加载模型
  20. train_info_args['seed'] = None
  21. parser = HfArgumentParser((ModelArguments, TrainingArguments, DataArguments, LoraArguments))
  22. model_args, training_args, data_args, _ = parser.parse_dict(train_info_args)
  23.  
  24. setup_model_profile()
  25.  
  26. dataHelper = NN_DataHelper(model_args, training_args, data_args)
  27. tokenizer: ChatGLMTokenizer
  28. tokenizer, _, _, _ = dataHelper.load_tokenizer_and_config(tokenizer_class_name=ChatGLMTokenizer, config_class_name=ChatGLMConfig)
  29.  
  30. config = ChatGLMConfig.from_pretrained('./best_ckpt')
  31. # config = ChatGLMConfig.from_pretrained('./best_ckpt')
  32. config.initializer_weight = False
  33.  
  34. lora_args = LoraArguments.from_pretrained('./last_ckpt')
  35. # lora_args = LoraArguments.from_pretrained('./best_ckpt')
  36.  
  37. assert lora_args.inference_mode == True and config.pre_seq_len is None
  38.  
  39. pl_model = MyTransformer(config=config, model_args=model_args, training_args=training_args, lora_args=lora_args)
  40. # 加载lora权重
  41. pl_model.backbone.from_pretrained(pl_model.backbone.model, pretrained_model_name_or_path='./last_ckpt',
  42.                                       lora_config=lora_args)
  43. # pl_model.backbone.from_pretrained(pl_model.backbone.model, pretrained_model_name_or_path = './best_ckpt', lora_config = lora_args)
  44.  
  45. model = pl_model.get_glm_model()
  46. # 按需修改
  47. model.half().cuda()
  48. model = model.eval()
  49.  
  50.  
  51. def infer(query,
  52.           history: Optional[List[Tuple]],
  53.           max_length, top_p, temperature):
  54.     # if cmd_opts.ui_dev:
  55.     #     return "hello", "hello, dev mode!"
  56.  
  57.     if not model:
  58.         raise "Model not loaded"
  59.  
  60.     if history is None:
  61.         history = []
  62.     output, history = model.chat(
  63.         tokenizer, query=query, history=history,
  64.         max_length=max_length,
  65.         top_p=top_p,
  66.         temperature=temperature
  67.     )
  68.     print(output)
  69.     torch_gc()
  70.     return query, output
  71.  
  72. def predict(query, max_length, top_p, temperature):
  73.     ctx.limit_round()
  74.     _, output = infer(
  75.         query=query,
  76.         history=ctx.history,
  77.         max_length=max_length,
  78.         top_p=top_p,
  79.         temperature=temperature
  80.     )
  81.     ctx.append(query, output)
  82.     torch_gc()
  83.     # for clear input textbox
  84.     return ctx.history, ""
  85.  
  86.  
  87. def clear_history():
  88.     ctx.clear()
  89.     return gr.update(value=[])
  90.  
  91.  
  92. def apply_max_round_click(max_round):
  93.     ctx.max_rounds = max_round
  94.  
  95. def reload_javascript():
  96.     scripts_list = [os.path.join(script_path, i) for i in os.listdir(script_path) if i.endswith(".js")]
  97.     javascript = ""
  98.     # with open("script.js", "r", encoding="utf8") as js_file:
  99.     #     javascript = f'<script>{js_file.read()}</script>'
  100.  
  101.     for path in scripts_list:
  102.         with open(path, "r", encoding="utf8") as js_file:
  103.             javascript += f"\n<script>{js_file.read()}</script>"
  104.  
  105.     # todo: theme
  106.     # if cmd_opts.theme is not None:
  107.     #     javascript += f"\n<script>set_theme('{cmd_opts.theme}');</script>\n"
  108.  
  109.     def template_response(*args, **kwargs):
  110.         res = _gradio_template_response_orig(*args, **kwargs)
  111.         res.body = res.body.replace(
  112.             b'</head>', f'{javascript}</head>'.encode("utf8"))
  113.         res.init_headers()
  114.         return res
  115.  
  116.     gr.routes.templates.TemplateResponse = template_response
  117.  
  118. def main():
  119.     # 创建ui
  120.     reload_javascript()
  121.  
  122.     with gr.Blocks(css=css, analytics_enabled=False) as chat_interface:
  123.         prompt = "输入你的内容..."
  124.         with gr.Row():
  125.             with gr.Column(scale=3):
  126.                 gr.Markdown("""<h2><center>ChatGLM WebUI</center></h2>""")
  127.                 with gr.Row():
  128.                     with gr.Column(variant="panel"):
  129.                         with gr.Row():
  130.                             max_length = gr.Slider(minimum=4, maximum=4096, step=4, label='Max Length', value=2048)
  131.                             top_p = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Top P', value=0.7)
  132.                         with gr.Row():
  133.                             temperature = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Temperature',
  134.                                                     value=0.95)
  135.  
  136.                         with gr.Row():
  137.                             max_rounds = gr.Slider(minimum=1, maximum=100, step=1, label="最大对话轮数（调小可以显著改善爆显存，但是会丢失上下文）",
  138.                                                    value=20)
  139.                             apply_max_rounds = gr.Button("✔", elem_id="del-btn")
  140.  
  141.                 with gr.Row():
  142.                     with gr.Column(variant="panel"):
  143.                         with gr.Row():
  144.                             clear = gr.Button("清空对话（上下文）")
  145.  
  146.                         with gr.Row():
  147.                             save_his_btn = gr.Button("保存对话")
  148.                             load_his_btn = gr.UploadButton("读取对话", file_types=['file'], file_count='single')
  149.  
  150.             with gr.Column(scale=7):
  151.                 chatbot = gr.Chatbot(elem_id="chat-box", show_label=False).style(height=800)
  152.                 with gr.Row():
  153.                     input_message = gr.Textbox(placeholder=prompt, show_label=False, lines=2, elem_id="chat-input")
  154.                     clear_input = gr.Button("
    声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/很楠不爱3/article/detail/405446
    推荐阅读
    相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。