1. 文本预处理
文本预处理是 NLP 的第一步，包括去除标点符号、转换为小写、分词等操作。
import re
import string
def preprocess_text(text):
# 去除标点符号
text = text.translate(str.maketrans('', '', string.punctuation))
# 转换为小写
text = text.lower()
# 分词
words = text.split()
return words
text = "Hello, World! This is a test."
preprocessed_text = preprocess_text(text)
print(preprocessed_text) # 输出: ['hello', 'world', 'this', 'is', 'a', 'test']2. 词干提取和词形还原
词干提取(Stemming)和词形还原(Lemmatization)是将单词还原为其基本形式的技术。
from nltk.stem import PorterStemmer, WordNetLemmatizer
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
words = ["running", "jumps", "better", "worse"]
stemmed_words = [stemmer.stem(word) for word in words]
lemmatized_words = [lemmatizer.lemmatize(word) for word in words]
print(stemmed_words) # 输出: ['run', 'jump', 'better', 'worst']
print(lemmatized_words) # 输出: ['run', 'jump', 'better', 'worse']3. 停用词去除
停用词(Stop Words)是指在文本中频繁出现但对语义贡献不大的词汇，如“the”、“is”等。
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))
def remove_stopwords(words):
filtered_words = [word for word in words if word not in stop_words]
return filtered_words
words = ["this", "is", "a", "test", "of", "stop", "words"]
filtered_words = remove_stopwords(words)
print(filtered_words) # 输出: ['test', 'stop', 'words']4. 词频统计
词频统计可以帮助我们了解文本中最常见的词汇。
from collections import Counter
words = ["apple", "banana", "apple", "orange", "banana", "banana"]
word_counts = Counter(words)
print(word_counts) # 输出: Counter({'banana': 3, 'apple': 2, 'orange': 1})5. TF-IDF 计算
TF-IDF(Term Frequency-Inverse Document Frequency)是一种统计方法，用于评估一个词在一个文档或一组文档中的重要性。
from sklearn.feature_extraction.text import TfidfVectorizer
documents = [
"The cat in the hat.",
"A cat is a fine pet.",
"Dogs and cats make good pets."
]
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)
print(tfidf_matrix.toarray())
# 输出: [[0. 0.57735027 0.57735027 0.57735027 0. 0. ]
# [0.57735027 0.57735027 0.57735027 0. 0. 0. ]
# [0.40824829 0.40824829 0. 0. 0.57735027 0.57735027]]from sklearn.feature_extraction.text import TfidfVectorizer
documents = [
"The cat in the hat.",
"A cat is a fine pet.",
"Dogs and cats make good pets."
]
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)
print(tfidf_matrix.toarray())
# 输出: [[0. 0.57735027 0.57735027 0.57735027 0. 0. ]
# [0.57735027 0.57735027 0.57735027 0. 0. 0. ]
# [0.40824829 0.40824829 0. 0. 0.57735027 0.57735027]]6. 命名实体识别(NER)
命名实体识别(NER)是从文本中识别出特定类型的实体，如人名、地名、组织名等。
import spacy
nlp = spacy.load("en_core_web_sm")
text = "Apple is looking at buying U.K. startup for $1 billion"
doc = nlp(text)
for ent in doc.ents:
print(ent.text, ent.label_)
# 输出:
# Apple ORG
# U.K. GPE
# $1 billion MONEY7. 情感分析
情感分析是判断文本的情感倾向，如正面、负面或中性。
from textblob import TextBlob
text = "I love this movie!"
blob = TextBlob(text)
sentiment = blob.sentiment.polarity
print(sentiment) # 输出: 0.8758. 文本分类
文本分类是将文本归类到预定义的类别中，如垃圾邮件检测、新闻分类等。
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
# 加载数据集
data = fetch_20newsgroups(subset='all')
# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.2, random_state=42)
# 创建管道
pipeline = Pipeline([
('vectorizer', CountVectorizer()),
('classifier', MultinomialNB())
])
# 训练模型
pipeline.fit(X_train, y_train)
# 预测
y_pred = pipeline.predict(X_test)
# 评估
print(classification_report(y_test, y_pred, target_names=data.target_names))9. 文本聚类
文本聚类是将相似的文本分组到同一个类别中。
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
documents = [
"The cat in the hat.",
"A cat is a fine pet.",
"Dogs and cats make good pets.",
"I love my pet dog.",
"My dog loves to play with cats."
]
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)
kmeans = KMeans(n_clusters=2)
kmeans.fit(tfidf_matrix)
labels = kmeans.labels_
print(labels) # 输出: [0 0 1 1 1]10. 机器翻译
机器翻译是将一种语言的文本自动翻译成另一种语言。
from transformers import MarianMTModel, MarianTokenizer
model_name = "Helsinki-NLP/opus-mt-en-de"
tokenizer = MarianTokenizer.from_pretrained(model_name)
model = MarianMTModel.from_pretrained(model_name)
text = "I love this movie!"
translated = model.generate(**tokenizer(text, return_tensors="pt", padding=True))
translated_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]
print(translated_text) # 输出: ['Ich liebe diesen Film!']11. 问答系统
问答系统是根据给定的上下文回答问题。
from transformers import pipeline
qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")
context = "The Transformer is a deep learning model introduced in 2017 by Vaswani et al. that revolutionized natural language processing."
question = "What year was the Transformer model introduced?"
answer = qa_pipeline(question=question, context=context)
print(answer['answer']) # 输出: 201712. 文本生成
文本生成是使用模型生成新的文本，如生成诗歌、故事等。
from transformers import GPT2LMHeadModel, GPT2Tokenizer
model_name = "gpt2"
tokenizer = GPT2Tokenizer.from_pretrained(model_name)
model = GPT2LMHeadModel.from_pretrained(model_name)
prompt = "Once upon a time"
input_ids = tokenizer.encode(prompt, return_tensors="pt")
output = model.generate(input_ids, max_length=50, num_return_sequences=1)
generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
print(generated_text)
# 输出: Once upon a time in a land far away, there lived a brave knight who set out on a quest to find a magical treasure.实战案例：垃圾邮件检测
假设你是一家电子邮件服务提供商，需要开发一个系统来检测并过滤垃圾邮件。我们将使用朴素贝叶斯分类器来实现这个任务。
数据准备
首先，我们需要准备一些训练数据。这里我们使用 sklearn 提供的 20newsgroups 数据集的一部分作为示例。
from sklearn.datasets import fetch_20newsgroups
# 加载数据集
data = fetch_20newsgroups(subset='all', categories=['alt.atheism', 'soc.religion.christian'])
# 将数据划分为训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.2, random_state=42)特征提取
使用 CountVectorizer 将文本转换为特征向量。
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer()
X_train_vectorized = vectorizer.fit_transform(X_train)
X_test_vectorized = vectorizer.transform(X_test)模型训练
使用朴素贝叶斯分类器进行训练。
from sklearn.naive_bayes import MultinomialNB
model = MultinomialNB()
model.fit(X_train_vectorized, y_train)模型评估
评估模型的性能。
from sklearn.metrics import classification_report
y_pred = model.predict(X_test_vectorized)
print(classification_report(y_test, y_pred, target_names=data.target_names))应用
现在我们可以使用训练好的模型来检测新的邮件是否为垃圾邮件。
def predict_spam(email_text):
email_vectorized = vectorizer.transform([email_text])
prediction = model.predict(email_vectorized)
return "Spam" if prediction[0] == 1 else "Not Spam"
email_text = "Get rich quick! Click here to win a million dollars!"
print(predict_spam(email_text)) # 输出: Spam总结
本文介绍了 12 个实用的自然语言处理(NLP)案例，涵盖了文本预处理、词干提取、词形还原、停用词去除、词频统计、TF-IDF 计算、命名实体识别、情感分析、文本分类、文本聚类、机器翻译、问答系统和文本生成。通过这些案例，你可以更好地理解和应用 NLP 技术。
