2025-05-20 模型下载--文本向量化--Faiss检索

模型下载

• 使用Python脚本进行下载

from huggingface_hub import snapshot_download
# import os# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
# 自定义下载目录（Windows 路径建议用 raw string 或 pathlib）
download_dir = r"../model/V2/jina-embeddings-v3"local_dir = snapshot_download(repo_id="jinaai/jina-embeddings-v3",local_dir=download_dir,local_dir_use_symlinks=False  # 避免 Windows 下软链接问题
)print("模型保存路径：", local_dir)

• 当使用跑数据集的时候，还是会出现错误，找不到各种文件，原因是依赖了其他的文件
• 地址 https://huggingface.co/jinaai/xlm-roberta-flash-implementation/tree/main

文本向量化

import pandas as pd
from sentence_transformers import SentenceTransformer
import os# =========================
# 📍 Step 1: 加载数据
# =========================
input_datapath = './data/fine_food_reviews_1k.csv'df = pd.read_csv(input_datapath, index_col=0)
df = df[["Time", "ProductId", "UserId", "Score", "Summary", "Text"]]
df = df.dropna()# 合并文本字段
df["combined"] = ("Title: " + df.Summary.str.strip() + "; Content: " + df.Text.str.strip()
)# =========================
# 📍 Step 2: 加载本地模型
# =========================
model_path = 'model/jina-embeddings-v3/models--jinaai--jina-embeddings-v3/snapshots/f1944de8402dcd5f2b03f822a4bc22a7f2de2eb9'
model = SentenceTransformer(model_path, trust_remote_code=True)# =========================
# 📍 Step 3: 批量编码文本
# =========================
texts = df["combined"].tolist()# tqdm 显示进度条，convert_to_numpy 提供 array 输出
embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True)# =========================
# 📍 Step 4: 存储向量数据
# =========================
df["embedding"] = embeddings.tolist()# 保存为 parquet（高效）或 csv
output_path = './output/fine_food_reviews_with_embeddings.parquet'
os.makedirs(os.path.dirname(output_path), exist_ok=True)
df.to_parquet(output_path, index=False)print(f"✅ 向量化完成，共生成 {len(df)} 条记录，已保存至：{output_path}")

Faiss检索

import pandas as pd
import numpy as np
import faiss
from sentence_transformers import SentenceTransformer# ========== 1. 加载数据 ==========
df = pd.read_parquet('./output/fine_food_reviews_with_embeddings.parquet')
embeddings = np.array(df['embedding'].to_list())  # 转成 numpy array# ========== 2. 加载模型 ==========
model_path = 'model/jina-embeddings-v3/models--jinaai--jina-embeddings-v3/snapshots/f1944de8402dcd5f2b03f822a4bc22a7f2de2eb9'
model = SentenceTransformer(model_path, trust_remote_code=True)# ========== 3. 构建向量索引 ==========
index = faiss.IndexFlatL2(embeddings.shape[1])
index.add(embeddings)# ========== 4. 查询示例 ==========
query = "蜂蜜"
query_vec = model.encode([query], convert_to_numpy=True)# 查询 TopK 相似文本
D, I = index.search(query_vec, k=3)print("🔍 查询结果：")
for idx in I[0]:print(f"\n📌 Score: {df.iloc[idx]['Score']}")print(f"📄 Text: {df.iloc[idx]['combined']}")