python：使用 OpenAI CLIP 模型进行图像与文本的语义匹配，并用彩虹色带可视化 CLIP 模型的相似度矩阵

作者：CSDN @ _养乐多_

本文将介绍如何使用 OpenAI 的 CLIP 模型来实现图像与文本之间的语义匹配。代码使用 Python 语言，加载多个图像与类别文本，并通过计算余弦相似度判断每张图片最匹配的文本标签。

结果如下图所示，

---

---

一、什么是 CLIP？

CLIP（Contrastive Language–Image Pre-training）是由 OpenAI 提出的模型，它能够将图像和文本映射到同一个向量空间，从而实现跨模态的理解能力。CLIP 的强大之处在于它无需专门为某个任务训练，也能胜任广泛的图像识别和文本匹配任务。

二、准备工作

准备好一个包含若干图像的文件夹（如 ./images）。如"cat.jpg", “dog.jpg”, “car.jpg”, “mountain.jpg”, “food.jpg”。

安装第三方库

pip install torch torchvision ftfy regex tqdm
pip install git+https://github.com/openai/CLIP.git

三、代码

import torch
import clip
from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np
from typing import Listdef load_images(image_paths: List[str]) -> List[Image.Image]:"""加载并转换图像为RGB格式参数:image_paths: 图像文件路径列表返回:PIL Image对象列表"""return [Image.open(path).convert("RGB") for path in image_paths]def compute_similarity(model, preprocess, images, texts, device):"""使用CLIP模型计算图像和文本之间的余弦相似度参数:model: CLIP模型实例preprocess: 图像预处理函数images: PIL Image列表texts: 文本标签列表device: 设备名称（cpu或cuda）返回:numpy数组，形状为 (图像数, 文本类别数)，值为相似度分数"""# 预处理图像，转为tensor并堆叠到一个batch，放到指定设备image_tensors = torch.stack([preprocess(img) for img in images]).to(device)# 对文本进行tokenize并放到设备text_tokens = clip.tokenize(texts).to(device)with torch.no_grad():# 编码图像和文本特征image_features = model.encode_image(image_tensors)text_features = model.encode_text(text_tokens)# 对特征做归一化，方便计算余弦相似度image_features /= image_features.norm(dim=-1, keepdim=True)text_features /= text_features.norm(dim=-1, keepdim=True)# 计算余弦相似度矩阵 (图像特征 @ 文本特征转置)similarity = image_features @ text_features.Treturn similarity.cpu().numpy()def visualize_similarity_matrix_with_images(images, image_paths, categories, similarity):"""可视化相似度矩阵，左侧显示对应图片，右侧按类别显示彩虹色相似度色块，色块内标注相似度数值参数:images: PIL Image列表image_paths: 图像路径列表（用于显示文件名）categories: 文本类别标签列表similarity: numpy数组，图像与文本类别的相似度矩阵"""num_images = len(images)num_categories = len(categories)# 计算相似度最小和最大值，用于归一化色带映射min_val = similarity.min()max_val = similarity.max()# 创建子图，列数比类别数多1列，用于放图片fig, axs = plt.subplots(num_images, num_categories + 1, figsize=(2 * (num_categories + 1), 2 * num_images))if num_images == 1:axs = axs.reshape(1, -1)  # 保证axs是2D数组，方便统一处理# 去除子图间距，保证格子紧密无缝隙plt.subplots_adjust(wspace=0, hspace=0)# 第一行显示类别标题（只显示文字，不显示轴）for j in range(num_categories):axs[0, j+1].set_title(categories[j], fontsize=11, pad=6)axs[0, j+1].axis('off')axs[0, 0].axis('off')  # 左上角空白# 遍历每张图片和每个类别for i in range(num_images):# 左侧显示图片，不显示坐标轴axs[i, 0].imshow(images[i])axs[i, 0].axis('off')# 图片文件名作为标题axs[i, 0].set_title(os.path.basename(image_paths[i]), fontsize=8, pad=4)for j in range(num_categories):sim_val = similarity[i, j]# 将相似度归一化到0-1区间，用于颜色映射norm_val = (sim_val - min_val) / (max_val - min_val) if max_val > min_val else 0# 使用彩虹色带(rainbow)映射相似度color = plt.cm.rainbow(norm_val)# 显示彩色方块axs[i, j+1].imshow(np.ones((20,20,3)) * color[:3])axs[i, j+1].axis('off')# 根据颜色亮度选择文字颜色，保证对比度，易读性brightness = 0.299 * color[0] + 0.587 * color[1] + 0.114 * color[2]font_color = 'black' if brightness > 0.6 else 'white'# 在色块中心写入相似度数值axs[i, j+1].text(10, 10, f"{sim_val:.2f}", ha='center', va='center', fontsize=9, color=font_color)# 自动紧凑布局，去除多余边距plt.tight_layout(pad=0)plt.show()def main():# 选择设备，优先cuda，否则cpudevice = "cuda" if torch.cuda.is_available() else "cpu"# 加载CLIP模型和对应的图像预处理函数model, preprocess = clip.load("ViT-B/32", device=device)# 指定图像文件夹路径image_folder = "./images"# 遍历文件夹，筛选常见图片格式路径image_paths = [os.path.join(image_folder, f) for f in os.listdir(image_folder)if f.lower().endswith(('jpg', 'png', 'jpeg'))]# 读取所有图像images = load_images(image_paths)# 设定要匹配的文本类别categories = ["cat", "dog", "car", "mountain", "food"]# 计算图像与文本类别的相似度矩阵similarity = compute_similarity(model, preprocess, images, categories, device)# 输出每张图片匹配度最高的类别及相似度for i, path in enumerate(image_paths):best_idx = similarity[i].argmax()print(f"Image: {os.path.basename(path)} => Best match: '{categories[best_idx]}' (Similarity: {similarity[i][best_idx]:.4f})")# 可视化相似度矩阵visualize_similarity_matrix_with_images(images, image_paths, categories, similarity)if __name__ == "__main__":main()

谢谢各位读者对本人文章的关注和支持！

四、代码2

和第三部分一样，只是可视化效果不一样。

import torch
import clip
from PIL import Image
import os
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Listdef load_images(image_paths: List[str]) -> List[Image.Image]:return [Image.open(path).convert("RGB") for path in image_paths]def compute_similarity(model, preprocess, images, texts, device):image_tensors = torch.stack([preprocess(img) for img in images]).to(device)text_tokens = clip.tokenize(texts).to(device)with torch.no_grad():image_features = model.encode_image(image_tensors)text_features = model.encode_text(text_tokens)image_features /= image_features.norm(dim=-1, keepdim=True)text_features /= text_features.norm(dim=-1, keepdim=True)similarity = image_features @ text_features.Treturn similarity.cpu().numpy()def visualize_similarity_matrix(image_paths, categories, similarity):plt.figure(figsize=(12, max(4, len(image_paths) * 0.5)))sns.heatmap(similarity, xticklabels=categories, yticklabels=[os.path.basename(p) for p in image_paths],cmap="YlGnBu", annot=True, fmt=".2f")plt.xlabel("Categories")plt.ylabel("Images")plt.title("Image-Text Similarity Matrix")plt.tight_layout()plt.show()def main():device = "cuda" if torch.cuda.is_available() else "cpu"model, preprocess = clip.load("ViT-B/32", device=device)image_folder = "./images"image_paths = [os.path.join(image_folder, f) for f in os.listdir(image_folder)if f.lower().endswith(('jpg', 'png', 'jpeg'))]images = load_images(image_paths)categories = ["cat", "dog", "car", "mountain", "food"]similarity = compute_similarity(model, preprocess, images, categories, device)# 输出每张图片与文本类别的相似度for i, path in enumerate(image_paths):best_idx = similarity[i].argmax()print(f"Image: {os.path.basename(path)} => Best match: '{categories[best_idx]}' (Similarity: {similarity[i][best_idx]:.4f})")visualize_similarity_matrix(image_paths, categories, similarity)if __name__ == "__main__":main()