AI自然语言处理（NLP）核心技术完全教程
1. NLP技术概述与演进
自然语言处理（Natural Language Processing）是人工智能领域中最具挑战性的方向之一，其核心目标是让计算机理解、生成和推理人类语言。回顾NLP的发展历程，大致经历了以下几个关键阶段：
规则驱动时代（1950s-1990s）：早期NLP系统依赖手工编写的语法规则和知识库。典型的例子包括基于上下文无关文法（CFG）的句法分析器和专家系统。这类方法精度高但覆盖面窄，难以扩展。
统计学习时代（1990s-2013）：随着大规模语料的出现，隐马尔可夫模型（HMM）、条件随机场（CRF）、最大熵模型等统计方法成为主流。这些方法从数据中自动学习语言模式，显著提升了系统的泛化能力。
深度学习时代（2013-2018）：Word2Vec的出现开启了神经网络在NLP中的大规模应用。循环神经网络（RNN）、长短期记忆网络（LSTM）和注意力机制逐步取代了传统统计模型。
预训练大模型时代（2018至今）：BERT、GPT系列等预训练语言模型的出现，带来了NLP领域的范式转变——通过大规模无监督预训练加少量有监督微调的模式，几乎统一了所有NLP任务的建模方式。
理解这个演进脉络，有助于在实际项目中选择合适的技术路线。
2. 文本预处理：分词、词性标注与命名实体识别
文本预处理是所有NLP任务的基础。原始文本需要经过一系列清洗和结构化步骤，才能被模型有效利用。
2.1 中文分词
中文没有天然的词边界，分词是中文NLP的第一步。主流方案包括基于词典的分词和基于序列标注的神经网络分词。
import jieba
import jieba.posseg as pseg
# 基础分词
text = "自然语言处理是人工智能领域的重要方向"
words = jieba.lcut(text)
print(words)
# ['自然语言', '处理', '是', '人工智能', '领域', '的', '重要', '方向']
# 精确模式 vs 搜索引擎模式
words_search = jieba.lcut_for_search("自然语言处理是人工智能领域的重要方向")
print(words_search)
# ['自然', '语言', '自然语言', '处理', '是', '人工', '智能', '人工智能', '领域', '的', '重要', '方向']
# 自定义词典
jieba.add_word("大语言模型", freq=10000, pos="n")
jieba.load_userdict("custom_dict.txt")
对于精度要求更高的场景，可以使用基于Transformer的分词模型：
from transformers import AutoTokenizer, AutoModelForTokenClassification
import torch
# 使用预训练模型进行中文分词
model_name = "bert-base-chinese"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForTokenClassification.from_pretrained(model_name)
inputs = tokenizer("深度学习在自然语言处理中的应用", return_tensors="pt")
with torch.no_grad():
outputs = model(**inputs)
predictions = torch.argmax(outputs.logits, dim=-1)
2.2 词性标注
词性标注为每个词分配语法类别（名词、动词、形容词等），为下游的句法分析和语义理解提供基础信息。
# 使用jieba进行词性标注
words = pseg.lcut("深度学习推动了自然语言处理的发展")
for word, flag in words:
print(f"{word}/{flag}", end=" ")
# 深度/l 学习/v 推动/v 了/u 自然语言/n 的/u 发展/vn
# 使用spaCy进行英文词性标注
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("Transformer models have revolutionized NLP research")
for token in doc:
print(f"{token.text:15} {token.pos_:8} {token.tag_}")
2.3 命名实体识别（NER）
NER的目标是从文本中识别出具有特定意义的实体，如人名、地名、组织名、时间等。这是信息抽取的基础任务。
# 使用spaCy进行NER
import spacy
nlp = spacy.load("zh_core_web_sm")
doc = nlp("华为公司在深圳发布了新一代麒麟芯片，余承东出席了发布会")
for ent in doc.ents:
print(f"实体: {ent.text}, 类型: {ent.label_}")
# 实体: 华为, 类型: ORG
# 实体: 深圳, 类型: LOC
# 实体: 余承东, 类型: PER
# 使用Hugging Face进行自定义NER
from transformers import pipeline
ner_pipeline = pipeline("ner", model="bert-base-chinese", aggregation_strategy="simple")
results = ner_pipeline("张三在北京大学攻读博士学位")
for entity in results:
print(f"{entity['word']}: {entity['entity_group']} ({entity['score']:.3f})")
3. 词嵌入技术：Word2Vec、GloVe与FastText
词嵌入将离散的词语映射到连续的向量空间，使得语义相近的词在向量空间中距离相近。这是深度学习NLP的基石。
3.1 Word2Vec
Word2Vec通过预测上下文（Skip-gram）或根据上下文预测目标词（CBOW）来学习词向量。
from gensim.models import Word2Vec
# 准备语料（已分词的句子列表）
sentences = [
["深度", "学习", "是", "人工智能", "的", "核心"],
["自然语言", "处理", "属于", "人工智能", "领域"],
["卷积", "神经网络", "在", "计算机", "视觉", "中", "应用", "广泛"],
# ... 实际场景中需要大量语料
]
# 训练Word2Vec模型
model = Word2Vec(
sentences,
vector_size=128, # 向量维度
window=5, # 上下文窗口大小
min_count=1, # 最低词频
sg=1, # 1=Skip-gram, 0=CBOW
workers=4,
epochs=10
)
# 获取词向量
vector = model.wv["深度"]
print(f"词向量维度: {vector.shape}")
# 查找相似词
similar = model.wv.most_similar("人工智能", topn=5)
for word, score in similar:
print(f"{word}: {score:.4f}")
# 词向量运算：king - man + woman ≈ queen
result = model.wv.most_similar(positive=["深度", "视觉"], negative=["语言"], topn=3)
3.2 GloVe
GloVe（Global Vectors）基于全局词共现矩阵进行分解，结合了全局统计信息和局部上下文窗口的优势。
import numpy as np
# 加载预训练GloVe向量
def load_glove(path, dim=100):
embeddings = {}
with open(path, 'r', encoding='utf-8') as f:
for line in f:
values = line.split()
word = values[0]
vector = np.asarray(values[1:], dtype='float32')
embeddings[word] = vector
return embeddings
glove = load_glove("glove.6B.100d.txt")
# 计算词相似度
def cosine_similarity(v1, v2):
return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
sim = cosine_similarity(glove["king"] - glove["man"] + glove["woman"], glove["queen"])
print(f"king - man + woman vs queen: {sim:.4f}") # 约 0.78
3.3 FastText
FastText在Word2Vec基础上引入了子词（subword）信息，能够处理未登录词（OOV），对形态丰富的语言尤其有效。
from gensim.models import FastText
# 训练FastText模型
ft_model = FastText(
sentences,
vector_size=128,
window=5,
min_count=1,
min_n=3, # 最小n-gram长度
max_n=6, # 最大n-gram长度
sg=1
)
# FastText能为未登录词生成向量
# 即使"Transformer架构"不在词表中，也能通过子词组合得到向量
vector = ft_model.wv["Transformer架构"]
print(f"未登录词向量维度: {vector.shape}")
4. 预训练语言模型：BERT、GPT与T5
预训练语言模型是当前NLP的核心技术。通过在海量文本上进行无监督预训练，模型学习到了丰富的语言知识。
4.1 BERT：双向编码器
BERT（Bidirectional Encoder Representations from Transformers）通过掩码语言模型（MLM）和下一句预测（NSP）任务进行预训练，能够获取双向上下文信息。
from transformers import BertTokenizer, BertModel
import torch
# 加载BERT模型
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
model = BertModel.from_pretrained("bert-base-chinese")
# 编码文本
text = "自然语言处理正在改变世界"
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)
with torch.no_grad():
outputs = model(**inputs)
# 获取隐藏状态：shape = (batch_size, seq_len, hidden_size)
hidden_states = outputs.last_hidden_state
print(f"序列长度: {hidden_states.shape[1]}, 隐藏维度: {hidden_states.shape[2]}")
# 获取[CLS]向量用于句子级任务
cls_vector = outputs.pooler_output
print(f"CLS向量维度: {cls_vector.shape}") # (1, 768)
4.2 GPT：自回归生成模型
GPT系列采用自回归方式（从左到右预测下一个token），天然适合文本生成任务。
from transformers import GPT2LMHeadModel, GPT2Tokenizer
# 使用GPT2进行文本生成
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model = GPT2LMHeadModel.from_pretrained("gpt2")
prompt = "Artificial intelligence will"
input_ids = tokenizer.encode(prompt, return_tensors="pt")
# 自回归生成
output = model.generate(
input_ids,
max_length=100,
num_beams=5, # 束搜索
temperature=0.7, # 控制随机性
top_p=0.9, # 核采样
do_sample=True,
repetition_penalty=1.2
)
generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
print(generated_text)
4.3 T5：文本到文本统一框架
T5将所有NLP任务统一为"文本到文本"的格式，通过不同的任务前缀来区分任务类型。
from transformers import T5ForConditionalGeneration, T5Tokenizer
tokenizer = T5Tokenizer.from_pretrained("t5-base")
model = T5ForConditionalGeneration.from_pretrained("t5-base")
# 文本摘要
input_text = "summarize: The Transformer model has become the dominant architecture for NLP tasks. It uses self-attention mechanisms to process sequences in parallel, replacing recurrent networks."
input_ids = tokenizer.encode(input_text, return_tensors="pt", max_length=512, truncation=True)
summary_ids = model.generate(input_ids, max_length=50, num_beams=4, early_stopping=True)
summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)
print(f"摘要: {summary}")
# 翻译
input_text = "translate English to German: The weather is beautiful today."
input_ids = tokenizer.encode(input_text, return_tensors="pt")
output = model.generate(input_ids, max_length=50)
translation = tokenizer.decode(output[0], skip_special_tokens=True)
print(f"翻译: {translation}")
5. 文本分类与情感分析
文本分类是NLP中最常见的任务之一。情感分析作为其重要子类，在商业场景中应用广泛。
5.1 基于BERT的文本分类
import torch
from torch import nn
from transformers import BertTokenizer, BertModel
class BertClassifier(nn.Module):
def __init__(self, num_classes, model_name="bert-base-chinese"):
super().__init__()
self.bert = BertModel.from_pretrained(model_name)
self.dropout = nn.Dropout(0.3)
self.classifier = nn.Linear(self.bert.config.hidden_size, num_classes)
def forward(self, input_ids, attention_mask):
outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
cls_output = outputs.pooler_output # [CLS] token
cls_output = self.dropout(cls_output)
return self.classifier(cls_output)
# 初始化分类器（例如3分类：正面、中性、负面）
model = BertClassifier(num_classes=3)
tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
# 推理示例
text = "这家餐厅的菜品非常美味，服务也很周到"
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=128)
with torch.no_grad():
logits = model(inputs["input_ids"], inputs["attention_mask"])
probs = torch.softmax(logits, dim=-1)
pred = torch.argmax(probs, dim=-1).item()
labels = ["负面", "中性", "正面"]
print(f"情感: {labels[pred]}, 概率: {probs[0][pred]:.4f}")
5.2 使用Hugging Face快速实现
from transformers import pipeline
# 零样本分类
classifier = pipeline("zero-shot-classification", model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli")
result = classifier(
"新款手机拍照效果惊人，电池续航也很持久",
candidate_labels=["科技产品", "美食评论", "体育新闻", "情感表达"]
)
print(f"分类结果: {result['labels'][0]}, 置信度: {result['scores'][0]:.4f}")
# 情感分析
sentiment = pipeline("sentiment-analysis")
result = sentiment("I absolutely love this product, it exceeded all my expectations!")
print(result) # [{'label': 'POSITIVE', 'score': 0.9998}]
6. 信息抽取与关系抽取
信息抽取的目标是从非结构化文本中提取结构化信息，包括实体抽取、关系抽取和事件抽取。
6.1 关系抽取
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch
# 基于BERT的关系抽取
# 输入格式：[CLS] 实体1 [SEP] 实体2 [SEP] 文本 [SEP]
def extract_relation(text, entity1, entity2, model, tokenizer):
prompt = f"{entity1} [SEP] {entity2} [SEP] {text}"
inputs = tokenizer(prompt, return_tensors="pt", max_length=512, truncation=True)
with torch.no_grad():
outputs = model(**inputs)
probs = torch.softmax(outputs.logits, dim=-1)
return probs
# 定义关系标签
relation_labels = ["无关系", "属于", "创始人", "位于", "发布", "合作"]
# 示例：从文本中抽取"华为-余承东"的关系
text = "余承东作为华为消费者业务CEO，多次在发布会上介绍新产品"
probs = extract_relation(text, "华为", "余承东", model, tokenizer)
pred_idx = torch.argmax(probs, dim=-1).item()
print(f"关系: {relation_labels[pred_idx]}")
6.2 基于Prompt的信息抽取
大模型时代，利用Prompt进行信息抽取成为主流方案：
from transformers import pipeline
# 使用大模型进行信息抽取
generator = pipeline("text2text-generation", model="google/flan-t5-large")
text = """
特斯拉CEO埃隆·马斯克于2024年3月宣布，公司将投资50亿美元
在德克萨斯州奥斯汀建设新的超级工厂。
"""
prompt = f"""Extract the following information from the text in JSON format:
- Organization
- Person
- Position
- Amount
- Location
- Date
Text: {text}
Information:"""
result = generator(prompt, max_length=200, do_sample=False)
print(result[0]["generated_text"])
7. 文本摘要：抽取式与生成式
7.1 抽取式摘要
抽取式方法直接从原文中选择关键句子组成摘要，保留了原文的表述准确性。
from transformers import pipeline
# 使用BERT进行抽取式摘要
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
text = """
Natural language processing has undergone significant transformations in recent years.
The introduction of transformer-based models has fundamentally changed how we approach
text understanding and generation. BERT, released in 2018, demonstrated that pre-training
on large corpora could dramatically improve performance on downstream tasks. GPT models
further showed that language models could be effective few-shot learners. These advances
have led to practical applications in machine translation, text summarization, question
answering, and dialogue systems.
"""
summary = summarizer(text, max_length=60, min_length=20, do_sample=False)
print(f"摘要: {summary[0]['summary_text']}")
7.2 生成式摘要
生成式方法通过编码器-解码器架构生成新的摘要文本，表达更加流畅自然。
from transformers import T5ForConditionalGeneration, T5Tokenizer
class AbstractiveSummarizer:
def __init__(self, model_name="csebuetnlp/mT5-large-sum"):
self.tokenizer = T5Tokenizer.from_pretrained(model_name)
self.model = T5ForConditionalGeneration.from_pretrained(model_name)
def summarize(self, text, max_length=150):
input_text = f"summarize: {text}"
inputs = self.tokenizer.encode(
input_text, return_tensors="pt",
max_length=512, truncation=True
)
summary_ids = self.model.generate(
inputs,
max_length=max_length,
min_length=30,
num_beams=4,
length_penalty=2.0,
early_stopping=True,
no_repeat_ngram_size=3
)
return self.tokenizer.decode(summary_ids[0], skip_special_tokens=True)
# 使用示例
summarizer = AbstractiveSummarizer()
text = "此处放入需要摘要的长文本..."
# summary = summarizer.summarize(text)
8. 机器翻译技术
8.1 基于Transformer的翻译模型
from transformers import MarianMTModel, MarianTokenizer
# 英译中
model_name = "Helsinki-NLP/opus-mt-en-zh"
tokenizer = MarianTokenizer.from_pretrained(model_name)
model = MarianMTModel.from_pretrained(model_name)
text = "The rapid advancement of artificial intelligence is reshaping industries worldwide."
inputs = tokenizer(text, return_tensors="pt", padding=True)
translated = model.generate(**inputs)
result = tokenizer.decode(translated[0], skip_special_tokens=True)
print(f"翻译: {result}")
# 批量翻译
texts = [
"Machine learning models require large amounts of training data.",
"Natural language processing enables computers to understand human language.",
"Deep learning has achieved remarkable results in computer vision."
]
inputs = tokenizer(texts, return_tensors="pt", padding=True, truncation=True, max_length=128)
translated = model.generate(**inputs)
results = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
for orig, trans in zip(texts, results):
print(f"原文: {orig}")
print(f"译文: {trans}\n")
8.2 使用大模型进行高质量翻译
from transformers import pipeline
# 使用NLLB（No Language Left Behind）进行多语言翻译
translator = pipeline("translation", model="facebook/nllb-200-distilled-600M",
src_lang="eng_Latn", tgt_lang="zho_Hans")
result = translator("Transformers have revolutionized the field of NLP.", max_length=100)
print(f"翻译: {result[0]['translation_text']}")
9. 大模型时代的NLP范式转变
大语言模型（LLM）的出现彻底改变了NLP的工作范式：
9.1 从微调到Prompt Engineering
from transformers import pipeline
# 零样本学习：无需训练数据
generator = pipeline("text2text-generation", model="google/flan-t5-large")
# 通过Prompt定义任务
prompts = {
"情感分析": "Classify the sentiment of this review as positive or negative: 'This movie was absolutely fantastic and I loved every minute of it.'",
"关键词提取": "Extract the key topics from this text: 'Cloud computing and edge AI are transforming enterprise IT infrastructure.'",
"文本改写": "Rewrite this sentence in a more formal tone: 'The new product is really cool and people will love it.'"
}
for task, prompt in prompts.items():
result = generator(prompt, max_length=100, do_sample=False)
print(f"{task}: {result[0]['generated_text']}")
9.2 RAG（检索增强生成）
from transformers import AutoTokenizer, AutoModel
import torch
import numpy as np
class SimpleRAG:
def __init__(self, model_name="sentence-transformers/all-MiniLM-L6-v2"):
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.model = AutoModel.from_pretrained(model_name)
self.documents = []
self.embeddings = []
def encode(self, texts):
inputs = self.tokenizer(texts, padding=True, truncation=True,
return_tensors="pt", max_length=256)
with torch.no_grad():
outputs = self.model(**inputs)
# 使用平均池化
embeddings = outputs.last_hidden_state.mean(dim=1)
return embeddings.numpy()
def add_documents(self, docs):
self.documents.extend(docs)
emb = self.encode(docs)
self.embeddings.append(emb)
def search(self, query, top_k=3):
query_emb = self.encode([query])
all_emb = np.vstack(self.embeddings)
# 余弦相似度
scores = np.dot(all_emb, query_emb.T).flatten()
top_indices = scores.argsort()[-top_k:][::-1]
return [(self.documents[i], scores[i]) for i in top_indices]
# 使用示例
rag = SimpleRAG()
knowledge_base = [
"Transformer架构由Vaswani等人在2017年提出，采用自注意力机制。",
"BERT使用掩码语言模型进行预训练，能够理解双向上下文。",
"GPT系列模型采用自回归方式生成文本，从左到右逐个预测token。",
"注意力机制的核心公式为Attention(Q,K,V) = softmax(QK^T/√d)V。",
]
rag.add_documents(knowledge_base)
results = rag.search("注意力机制是怎么工作的？")
for doc, score in results:
print(f"[{score:.4f}] {doc}")
10. 实战案例：构建完整的NLP Pipeline
下面展示一个完整的NLP处理流程，涵盖从原始文本到结构化输出的各个环节：
import jieba
import jieba.posseg as pseg
from transformers import pipeline
from collections import Counter
class NLPPipeline:
"""完整的中文NLP处理管道"""
def __init__(self):
# 初始化各组件
self.ner_pipeline = pipeline("ner", model="bert-base-chinese",
aggregation_strategy="simple")
self.sentiment_pipeline = pipeline("sentiment-analysis")
self.classifier = pipeline("zero-shot-classification",
model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli")
def preprocess(self, text):
"""文本清洗与分词"""
# 基本清洗
text = text.strip()
# 分词
words = jieba.lcut(text)
# 去停用词
stopwords = {"的", "了", "在", "是", "和", "与", "及", "等", "为"}
words = [w for w in words if w not in stopwords and len(w.strip()) > 0]
return words
def extract_entities(self, text):
"""命名实体识别"""
return self.ner_pipeline(text)
def analyze_sentiment(self, text):
"""情感分析"""
return self.sentiment_pipeline(text)[0]
def classify_topic(self, text, labels):
"""主题分类"""
result = self.classifier(text, candidate_labels=labels)
return list(zip(result["labels"], result["scores"]))
def keyword_extraction(self, text, top_k=10):
"""关键词提取（基于词频与词性）"""
words = pseg.lcut(text)
# 保留有意义的词性
valid_pos = {"n", "nr", "ns", "nt", "nz", "v", "vn", "a"}
keywords = [(w.word, w.flag) for w in words
if w.flag in valid_pos and len(w.word) > 1]
word_freq = Counter([w[0] for w in keywords])
return word_freq.most_common(top_k)
def process(self, text, topic_labels=None):
"""完整处理流程"""
result = {
"原始文本": text,
"分词结果": self.preprocess(text),
"实体识别": self.extract_entities(text),
"情感分析": self.analyze_sentiment(text),
"关键词": self.keyword_extraction(text),
}
if topic_labels:
result["主题分类"] = self.classify_topic(text, topic_labels)
return result
# 使用示例
pipeline_nlp = NLPPipeline()
text = """2024年，华为在深圳发布了全新的Mate 70系列手机。
该产品搭载了最新的麒麟芯片，在AI计算性能方面取得了重大突破。
分析师认为，这标志着国产手机芯片进入新纪元。"""
labels = ["科技产品", "金融财经", "体育赛事", "社会新闻"]
result = pipeline_nlp.process(text, topic_labels=labels)
for key, value in result.items():
print(f"\n【{key}】")
print(value)
11. 多语言NLP与低资源语言
11.1 多语言模型
from transformers import AutoTokenizer, AutoModel
import torch
# 使用多语言模型处理不同语言
model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)
def get_embedding(text):
inputs = tokenizer(text, return_tensors="pt", padding=True,
truncation=True, max_length=128)
with torch.no_grad():
outputs = model(**inputs)
return outputs.last_hidden_state.mean(dim=1)
# 跨语言语义相似度
texts = {
"en": "The weather is beautiful today",
"zh": "今天天气真好",
"ja": "今日はとてもいい天気です",
"de": "Das Wetter ist heute wunderschön"
}
embeddings = {lang: get_embedding(text) for lang, text in texts.items()}
# 计算跨语言相似度
for lang1 in embeddings:
for lang2 in embeddings:
if lang1 < lang2:
sim = torch.cosine_similarity(embeddings[lang1], embeddings[lang2])
print(f"{lang1} <-> {lang2}: {sim.item():.4f}")
11.2 低资源语言的应对策略
面对训练数据稀缺的语言，可以采用以下策略：
# 策略1：跨语言迁移学习
# 用高资源语言训练的模型，迁移到低资源语言
from transformers import XLMRobertaForSequenceClassification, XLMRobertaTokenizer
model = XLMRobertaForSequenceClassification.from_pretrained(
"xlm-roberta-base", num_labels=3
)
tokenizer = XLMRobertaTokenizer.from_pretrained("xlm-roberta-base")
# 策略2：数据增强
def back_translate_augment(text, translator, pivot_lang="en"):
"""回译数据增强：目标语言 -> 枢纽语言 -> 目标语言"""
# 中文 -> 英文 -> 中文
intermediate = translator(text, src_lang="zh", tgt_lang=pivot_lang)
augmented = translator(intermediate, src_lang=pivot_lang, tgt_lang="zh")
return augmented
# 策略3：少样本学习
def few_shot_classify(examples, query, generator):
"""基于示例的少样本分类"""
prompt = "根据以下示例进行分类：\n"
for text, label in examples:
prompt += f"文本：{text}\n分类：{label}\n"
prompt += f"文本：{query}\n分类："
return generator(prompt, max_length=50, do_sample=False)
# 策略4：Prompt模板 + 大模型
def zero_shot_with_llm(text, categories, llm_pipeline):
"""使用大模型进行零样本分类"""
prompt = f"""请将以下文本分类到给定类别中。
文本：{text}
类别：{', '.join(categories)}
分类结果："""
return llm_pipeline(prompt, max_length=20, do_sample=False)
总结
NLP技术从规则驱动到统计学习，再到深度学习和大模型时代，经历了深刻的范式变革。掌握文本预处理、词嵌入、预训练模型、信息抽取、文本分类等核心技术，是构建高质量NLP应用的基础。在大模型时代，Prompt Engineering、RAG、少样本学习等新范式正在降低NLP应用的门槛，但对底层原理的理解仍然是解决复杂问题的关键。
在实际项目中，建议从简单方案起步——先用预训练Pipeline验证可行性，再根据需求逐步定制模型。NLP的工程实践远比理论复杂，数据质量、领域适配和评估指标的选择往往决定了项目的成败。
