如何用Python轻松检测文本相似性：原理与方法

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

documents = ["This is the first document.", "This document is the second document.", "And this is the third one."]
vectorizer = CountVectorizer()
X = vectorizer.fit_transform(documents)
cosine_sim = cosine_similarity(X, X)
print(cosine_sim)

def jaccard_similarity(set1, set2):
    intersection = len(set1.intersection(set2))
    union = len(set1.union(set2))
    return intersection / union

text1 = set("This is the first document.".split())
text2 = set("This document is the second document.".split())
similarity = jaccard_similarity(text1, text2)
print(similarity)

import nltk
from nltk.metrics import edit_distance

str1 = "kitten"
str2 = "sitting"
distance = edit_distance(str1, str2)
print(distance)

from sklearn.feature_extraction.text import TfidfVectorizer

corpus = ["This is the first document.", "This document is the second document.", "And this is the third one."]
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(corpus)

from datasketch import MinHash, MinHashLSH

# 创建MinHash对象
m1 = MinHash()
m2 = MinHash()

# 添加元素到MinHash
for d in data1:
    m1.update(d.encode('utf8'))
for d in data2:
    m2.update(d.encode('utf8'))

# 创建MinHash LSH索引
lsh = MinHashLSH(threshold=0.5, num_perm=128)
lsh.insert("m2", m2)

# 查询相似的MinHash
result = lsh.query(m1)
print("Approximate Jaccard:", len(result) / float(len(m1)))

from datasketch import MinHash, MinHashLSH

# 创建MinHash对象和MinHash LSH索引
m1 = MinHash()
m2 = MinHash()
lsh = MinHashLSH(threshold=0.5, num_perm=128)  # threshold是相似性阈值

# 文本数据
data1 = ["apple", "banana", "cherry", "date"]
data2 = ["banana", "date", "fig", "grape"]

# 添加元素到MinHash
for d in data1:
    m1.update(d.encode('utf8'))
for d in data2:
    m2.update(d.encode('utf8'))

# 插入MinHash到LSH索引
lsh.insert("m2", m2)

# 查询相似的MinHash
result = lsh.query(m1)

# 计算相似性
similarity = len(result) / float(len(m1))

print("Approximate Jaccard Similarity:", similarity)

from sklearn.feature_extraction.text import TfidfVectorizer

corpus = ["This is the first document.", "This document is the second document.", "And this is the third one."]
vectorizer = TfidfVectorizer()
X = vectorizer.fit_transform(corpus)

from gensim.models import Word2Vec

sentences = [["this", "is", "the", "first", "sentence"],
             ["this", "is", "the", "second", "sentence"],
             ["is", "this", "the", "third", "sentence"]]
model = Word2Vec(sentences, vector_size=100, window=5, min_count=1, sg=0)

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# 示例文本数据
documents = [
    "Python is a popular programming language",
    "Java is another widely used language",
    "Programming languages are essential for software development",
    "Python and Java are both used in web development"
]

# 创建TF-IDF向量化器
tfidf_vectorizer = TfidfVectorizer()

# 将文本数据转化为TF-IDF向量
tfidf_matrix = tfidf_vectorizer.fit_transform(documents)

# 计算文档之间的余弦相似性
similarity_matrix = cosine_similarity(tfidf_matrix)

# 打印相似性矩阵
print("Similarity Matrix:")
print(similarity_matrix)

# 查找最相似的文档
most_similar = similarity_matrix.argsort()[:, -2]

# 打印最相似的文档
for i, doc_index in enumerate(most_similar):
    print(f"Document {i} is most similar to Document {doc_index} (Similarity Score: {similarity_matrix[i][doc_index]:.2f})")

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense

model = Sequential()
model.add(Embedding(input_dim=vocab_size, output_dim=embed_size, input_length=max_sequence_length))
model.add(Conv1D(filters=128, kernel_size=5, activation='relu'))
model.add(GlobalMaxPooling1D())
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

from tensorflow.keras.layers import LSTM

model = Sequential()
model.add(Embedding(input_dim=vocab_size, output_dim=embed_size, input_length=max_sequence_length))
model.add(LSTM(128))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

pip install transformers

from transformers import AutoTokenizer, AutoModel
import torch
from scipy.spatial.distance import cosine

# 加载预训练的BERT模型和分词器
model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# 示例文本数据
text1 = "Python is a popular programming language"
text2 = "Java is another widely used language"

# 对文本进行分词和编码
inputs1 = tokenizer(text1, return_tensors="pt", padding=True, truncation=True)
inputs2 = tokenizer(text2, return_tensors="pt", padding=True, truncation=True)

# 使用BERT模型获取文本嵌入
outputs1 = model(**inputs1)
outputs2 = model(**inputs2)

# 获取文本的嵌入向量
embedding1 = outputs1.last_hidden_state.mean(dim=1).detach().numpy()[0]
embedding2 = outputs2.last_hidden_state.mean(dim=1).detach().numpy()[0]

# 计算余弦相似度
similarity = 1 - cosine(embedding1, embedding2)

# 打印相似性分数
print("BERT Similarity:", similarity)