Datawhale AI 夏令营：用户洞察挑战赛 Notebook（2）

针对文本聚类优化

优化TF-IDF特征工程

# 调整ngram_range：设置为(1, 2)，捕捉单字和双字词（如“不错”“不满意”）。
# 限制特征数量：通过max_features=5000保留高信息密度特征，降低维度。
# 过滤低频/高频词：设置min_df=2（过滤仅出现1次的词）和max_df=0.8（过滤出现超过80%样本的通用词）。from sklearn.feature_extraction.text import TfidfVectorizertfidf = TfidfVectorizer(ngram_range=(1, 2),  # 包含单字和双字词max_features=5000,   # 最大特征数min_df=2,            # 最小文档频率（出现至少2次）max_df=0.8,          # 最大文档频率（不超过80%样本）token_pattern=r"\b\w+\b"  # 匹配单词边界（兼容中文）
)

动态选择最佳簇数 n_clusters

import numpy as np
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score# 提取TF-IDF特征
X = tfidf.fit_transform(comments_to_cluster)# 轮廓系数：寻找最高得分
best_k = 0
best_silhouette = -1for k in range(5, 9):kmeans = KMeans(n_clusters=k, random_state=42)labels = kmeans.fit_predict(X)score = silhouette_score(X, labels)if score > best_silhouette:best_silhouette = scorebest_k = k

改进聚类算法

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import Normalizer  # 新增归一化步骤
from sklearn.cluster import KMeans
from sklearn.pipeline import make_pipeline# 构建管道：TF-IDF → 归一化 → KMeans
kmeans_predictor = make_pipeline(TfidfVectorizer(tokenizer=jieba.lcut, # 分词器ngram_range=(1, 2),  # 包含单字和双字词max_features=5000,   # 最大特征数min_df=2,            # 最小文档频率（出现至少2次）max_df=0.8,          # 最大文档频率（不超过80%样本）token_pattern=r"\b\w+\b"  # 匹配单词边界（兼容中文）),Normalizer(norm="l2"),  # 归一化向量长度为1（L2范数）KMeans(n_clusters=best_k, random_state=42, n_init=10)  # 使用标准KMeans
)# 训练与预测（保持原有逻辑）
comments_data_clean = comments_data[comments_data["sentiment_category"].isin([1, 3])]
kmeans_predictor.fit(comments_data_clean["comment_text"])
kmeans_cluster_label = kmeans_predictor.predict(comments_data_clean["comment_text"])kmeans_top_word = []
tfidf_vectorizer = kmeans_predictor.named_steps['tfidfvectorizer']
kmeans_model = kmeans_predictor.named_steps['kmeans']
feature_names = tfidf_vectorizer.get_feature_names_out()
cluster_centers = kmeans_model.cluster_centers_
for i in range(kmeans_model.n_clusters):top_feature_indices = cluster_centers[i].argsort()[::-1]top_word = ' '.join([feature_names[idx] for idx in top_feature_indices[:top_n_words]])kmeans_top_word.append(top_word)comments_data.loc[comments_data["sentiment_category"].isin([1, 3]), "positive_cluster_theme"] = [kmeans_top_word[x] for x in kmeans_cluster_label]

提交得分