LLamaSharp加载llama.cpp转化好的模型

新建.net8控制台项目

安装依赖包

LLamaSharp和LLamaSharp.Backend.Cpu

准备好转化好的模型

没有的话参考这篇文章https://blog.csdn.net/qq_36437991/article/details/137248622

编写代码

using LLama;
using LLama.Common;
using LLama.Native;

namespace llamasharpstu01;

class Program
{
    static async Task Main(string[] args)
    {
        //模型量化
        // string srcPath = "<model.bin>";
        // string dstPath = "<model_q4_0.bin>";
        // // LLamaQuantizer.Quantize(srcPath, dstPath, "q4_0");
        // LLamaQuantizer.Quantize(srcPath, dstPath, LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0);
        //将其更改为您自己的模型路径。
        string modelPath = @"G:\Cpp\qwenchattest10.5b-q5_k_m.gguf";
        //modelPath = @"G:\Cpp\qwenctest10.5b.gguf";
        modelPath = @"G:\Cpp\qwenchat0.5b-q4_0.gguf";
        var parameters = new ModelParams(modelPath)
        {
            //最长的聊天时间作为记忆。
            ContextSize = 1024,
            //要卸载到GPU的层数。请根据您的GPU内存进行调整。
            GpuLayerCount = 5,
            Encoding = Encoding.UTF8
        };
        using var model = LLamaWeights.LoadFromFile(parameters);
        using var context = model.CreateContext(parameters);
        var executor = new InteractiveExecutor(context);

        //添加聊天记录作为提示，告诉AI如何行动。
        var chatHistory = new ChatHistory();
        // chatHistory.AddMessage(AuthorRole.System, "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.");
        // chatHistory.AddMessage(AuthorRole.User, "Hello, Bob.");
        // chatHistory.AddMessage(AuthorRole.Assistant, "Hello. How may I help you today?");

        ChatSession session = new(executor, chatHistory);

        InferenceParams inferenceParams = new InferenceParams()
        {
            //答案中出现的token不应超过256个。-1表示无限制
            MaxTokens = 256,
            AntiPrompts = new List<string> { "User:" }
        };

        Console.ForegroundColor = ConsoleColor.Yellow;
        Console.Write("The chat session has started.\nUser: ");
        Console.ForegroundColor = ConsoleColor.Green;
        string userInput = Console.ReadLine() ?? "";

        while (userInput != "exit")
        {
            await foreach (var text in session.ChatAsync(new ChatHistory.Message(AuthorRole.User, userInput), inferenceParams))
            {
                Console.ForegroundColor = ConsoleColor.White;
                Console.Write(text);
            }
            Console.WriteLine("");
            Console.ForegroundColor = ConsoleColor.Green;
            userInput = Console.ReadLine() ?? "";
        }
    }
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64