【全】AIGC 系统的核心技术：自然语言处理 (Natural Language Processing, NLP)（代码+部署）...

<code class="language-plain has-numbering hljs" id="code_id_0">flowchart TD
    A[输入文本] --> B[文本预处理]
    B --> C[词嵌入层]
    C --> D[序列模型]
    D --> E[注意力机制]
    E --> F[输出层]
    subgraph NLP 管道
        A --> B --> C --> D --> E --> F
    end
</code>

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.

<code class="language-plain has-numbering hljs" id="code_id_1">import torch
from transformers import GPT2LMHeadModel, GPT2Tokenizer

# 加载预训练的 GPT-2 模型和 tokenizer
model_name = "gpt2"
model = GPT2LMHeadModel.from_pretrained(model_name)
tokenizer = GPT2Tokenizer.from_pretrained(model_name)

def generate_text(prompt, max_length=50):
    # 将输入文本编码为 token ids
    input_ids = tokenizer.encode(prompt, return_tensors='pt')
    
    # 使用模型生成文本
    output = model.generate(input_ids, max_length=max_length, num_return_sequences=1)

    # 解码生成的 token ids 为文本
    generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
    return generated_text

# 示例使用
prompt = "Once upon a time"
generated_text = generate_text(prompt)
print(generated_text)
</code>

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.

import gensim
from gensim.models import Word2Vec
 
# 准备一些示例文本数据
sentences = [
    ['this', 'is', 'a', 'sample', 'sentence'],
    ['word2vec', 'is', 'an', 'embedding', 'technique'],
    ['natural', 'language', 'processing', 'involves', 'understanding', 'text']
]
 
# 训练Word2Vec模型
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)
 
# 保存和加载模型
model.save("word2vec.model")
model = Word2Vec.load("word2vec.model")
 
# 获取单词向量
vector = model.wv['word2vec']
print(vector)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.

import numpy as np
 
def load_glove_model(glove_file):
    with open(glove_file, 'r', encoding='utf-8') as f:
        glove_model = {}
        for line in f:
            split_line = line.split()
            word = split_line[0]
            embedding = np.array(split_line[1:], dtype=float)
            glove_model[word] = embedding
    return glove_model
 
glove_file = "path/to/glove.6B.100d.txt"  # 替换为实际路径
glove_model = load_glove_model(glove_file)
 
# 获取单词向量
vector = glove_model.get('word')
print(vector)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.

import torch
import torch.nn as nn
 
class Attention(nn.Module):
    def __init__(self, hidden_dim):
        super(Attention, self).__init__()
        self.attn = nn.Linear(hidden_dim, 1)
    
    def forward(self, encoder_outputs):
        attn_weights = torch.softmax(self.attn(encoder_outputs), dim=1)
        context_vector = torch.sum(attn_weights * encoder_outputs, dim=1)
        return context_vector, attn_weights
 
# 示例输入数据
hidden_dim = 128
batch_size = 32
seq_len = 10
 
encoder_outputs = torch.randn(batch_size, seq_len, hidden_dim)
attention_layer = Attention(hidden_dim)
context_vector, attn_weights = attention_layer(encoder_outputs)
 
print(context_vector.shape)  # (batch_size, hidden_dim)
print(attn_weights.shape)    # (batch_size, seq_len, 1)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.

import torch
from transformers import BertTokenizer, BertForSequenceClassification
 
# 加载预训练的 BERT 模型和 tokenizer
model_name = "bert-base-uncased"
tokenizer = BertTokenizer.from_pretrained(model_name)
model = BertForSequenceClassification.from_pretrained(model_name, num_labels=2)
 
# 示例文本
texts = ["This is a positive text.", "This is a negative text."]
 
# 将文本编码为 token ids
inputs = tokenizer(texts, return_tensors="pt", padding=True, truncation=True, max_length=512)
 
# 获取模型输出
outputs = model(**inputs)
logits = outputs.logits
 
# 获取预测结果
predictions = torch.argmax(logits, dim=-1)
print(predictions)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.

import openai
 
# 设置 API 密钥
openai.api_key = 'your-api-key'
 
prompt = "Once upon a time"
 
response = openai.Completion.create(
    engine="davinci",
    prompt=prompt,
    max_tokens=50
)
 
generated_text = response.choices[0].text.strip()
print(generated_text)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.

from transformers import T5Tokenizer, T5ForConditionalGeneration
 
# 加载预训练的 T5 模型和 tokenizer
model_name = "t5-small"
tokenizer = T5Tokenizer.from_pretrained(model_name)
model = T5ForConditionalGeneration.from_pretrained(model_name)
 
# 示例文本
text = "translate English to French: The house is wonderful."
 
# 将文本编码为 token ids
inputs = tokenizer.encode(text, return_tensors="pt")
 
# 生成翻译文本
outputs = model.generate(inputs, max_length=40, num_beams=4, early_stopping=True)
translated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(translated_text)

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.