基于SAM2的眼动数据跟踪2.1——修正目标消失的记录方式

目录

一、前言

二、修正：新增独立按钮实现json转.aois

三、修正save函数

四、修改SAM2视频跟踪后处理（可选）

---

一、前言

        基于SAM2的眼动数据跟踪2中的第2节我有这样的观点：

那能不能目标消失的话我就不写入这一帧的目标框呢？感觉上是不能，因为目标消失时你不写入，它就会认为“这个物体存在但是没移动”，所以它会“自动补充渲染物体没移动这段时间的目标框位置，其实就是在两个关键帧之间直接渲染第一个关键的目标框”，只有你写入了目标框并且把"IsActive"置为false，它才会认为“物体消失了”。它这种搞法感觉不是很合理，因为你直接用一个变量判断是否目标消失，然后目标消失直接记录上次最后出现的目标的目标框不就行了，没必要每一帧都记录上次最后出现的目标的目标框吧。

        但是后来我发现Tobii Pro Lab这软件的aois其实是没毛病的。

        打个比方，原先我以为是这样：第1秒出现，第2秒消失，第3秒消失，第4秒出现，那么第2秒和第3秒要分别用一个json记录第1秒的目标框（最后一次出现的目标框，即目标残影框）并都在json里面用一个is_ghost=True的flag记录这个目标框是一个残影，然后当时我就觉得这不是搞笑嘛，如果第2秒消失，一直到第10000秒也是消失，第10001秒才出现，那不就是有1万多帧都在重复记录第1秒的框（最后一次出现的目标框）。

        但是我现在才明白，其实只要记录消失的第一帧就可以了，比如第1秒出现，第2秒消失，第3秒消失，第10001秒出现，只要第2秒记录一个关键帧并且在.aois中is_active=False即可，第3秒到第10000秒都不用写入关键帧到.aois中，只要等目标出现，比如第10001秒出现再写入这个关键帧并且把is_active=True。

        按照这篇修改之后，生成的.aois就不会这么大，导入进去的时候也不至于卡死。可以看到，即使一开始理解不到位，也可以生成能导入软件的.aois，很多时候我们先去做，之后再优化，远比一开始什么都不做要好得多。

二、修正：新增独立按钮实现json转.aois

首先我们直接在SAM2视频跟踪模型那行按钮后面新增一个“重新生成Aois”的按钮，这样只要一点这个按钮就会把图片文件夹里面的所有json转换为.aois。再新增一个清除所有.json和.aois的按钮（省的每次想要重新跑一下都要手动删，但video_meta.json我们是不删的）。

anylabeling/services/auto_labeling/segment_anything_2_video.py

class SegmentAnything2Video(Model):"""Segmentation model using SegmentAnything2 for video processing.This class provides methods to perform image segmentation on video framesusing the SegmentAnything2 model. It supports interactive marking andtracking of objects across frames."""class Meta:"""Meta class to define required configurations and UI elements."""required_config_names = ["type","name","display_name","model_cfg","model_path",]widgets = ["output_label","output_select_combobox","button_add_point","button_remove_point","button_add_rect","button_clear","button_finish_object","button_auto_decode","button_reset_tracker",   "button_rebuild_aois",  #新增重新生成aois"button_clear_json_aois",  #新增清除所有json和aois"toggle_preserve_existing_annotations","mask_fineness_slider","mask_fineness_value_label",]output_modes = {"polygon": QCoreApplication.translate("Model", "Polygon"),"rectangle": QCoreApplication.translate("Model", "Rectangle"),"rotation": QCoreApplication.translate("Model", "Rotation"),}default_output_mode = "polygon"

然后在anylabeling/views/labeling/widgets/auto_labeling/auto_labeling.ui里面找到button_reset_tracker，复制增加两个item，然后改改name和string。

    <item><widget class="QPushButton" name="button_reset_tracker"><property name="text"><string>Reset Tracker</string></property></widget></item><item><item><widget class="QPushButton" name="button_rebuild_aois"><property name="text"><string>重新生成Aois</string></property></widget></item><item><item><widget class="QPushButton" name="button_clear_json_aois"><property name="text"><string>删除Json和AOIS</string></property></widget></item><item>

然后在anylabeling/views/labeling/widgets/auto_labeling/auto_labeling.py中搜索button_reset_tracker，同样是照抄button_reset_tracker的样式代码，改一下触发函数

        # --- Configuration for: button_reset_tracker ---self.button_reset_tracker.setStyleSheet(get_normal_button_style())self.button_reset_tracker.clicked.connect(self.on_reset_tracker)# --- Configuration for: button_reset_tracker ---self.button_rebuild_aois.setStyleSheet(get_normal_button_style())self.button_rebuild_aois.clicked.connect(self.on_rebuild_aois)# --- Configuration for: button_reset_tracker ---self.button_clear_json_aois.setStyleSheet(get_normal_button_style())self.button_clear_json_aois.clicked.connect(self.on_clear_annotations)

 def hide_labeling_widgets(self):"""Hide labeling widgets by default"""widgets = ["button_run","button_add_point","button_remove_point","button_add_rect","button_clear","button_finish_object","button_send","edit_text","edit_conf","edit_iou","input_box_thres","input_conf","input_iou","output_label","output_select_combobox","toggle_preserve_existing_annotations","button_set_api_token","button_reset_tracker","button_clear_json_aois",# 新增"upn_select_combobox","gd_select_combobox","florence2_select_combobox","remote_server_select_combobox","button_auto_decode","button_skip_detection","mask_fineness_slider","mask_fineness_value_label",]for widget in widgets:getattr(self, widget).hide()

搜索on_reset_tracker，触发函数也模仿写一下，我们把触发函数要干的话转交给LabelWidget 干活

    def on_reset_tracker(self):"""Handle reset tracker"""self.model_manager.set_auto_labeling_reset_tracker()def on_rebuild_aois(self):"""按钮入口：直接让 LabelWidget 干活"""self.parent.rebuild_aoisdef on_clear_annotations(self):self.pareent.clear_annotations()

在anylabeling/views/labeling/label_widget.py中：

def rebuild_aois(self):import glob"""按钮：一键重新生成 output.aois"""json_dir = os.path.dirname(self.image_path)from anylabeling.views.labeling.label_file import LabelFilelf = LabelFile()out = lf.rebuild_aois(json_dir, fps=self.fps, total_us=self.total_us,image_width=self.image.width(), image_height=self.image.height())logger.info(f"Rebuild complete -> {out}")def clear_annotations(self):import globif not self.image_path:QMessageBox.warning(self, "Warning", "No file load!")return"""删除当前图片/视频所在目录下所有 *.json 与 *.aois"""dir_path = os.path.dirname(self.image_path)# 让用户先确认reply = QMessageBox.question(self, "Confirm",f"删除{dir_path}下面的所有.json和.aois并重新加载图片?(video_meta.json不会被删除)",QMessageBox.Yes | QMessageBox.No, QMessageBox.No)if reply != QMessageBox.Yes:return# 先关闭当前文件，避免占用self.reset_state()  # 删文件for ext in ("*.json", "*.aois"):for f in glob.glob(os.path.join(dir_path, ext)):if os.path.basename(f) == "video_meta.json":continue          # 跳过os.remove(f)logger.info(f"removed {f}")for idx in range(self.file_list_widget.count()):item = self.file_list_widget.item(idx)item.setCheckState(Qt.Unchecked)if self.canvas.shapesself.canvas.shapes.clear()self.canvas.update()current_index = self.fn_to_index[str(self.filename)]filename = self.image_list[current_index]if filename:self.load_file(filename)

在anylabeling\views\labeling\label_file.py中，

def extract_frame_number(self, filename):m = re.search(r'frame_(\d+)\.json', filename)return int(m.group(1)) if m else Nonedef rebuild_aois(self, json_dir, fps=None, total_us=None, image_width=None,         image_height=None): """把 json_dir 下所有 *.json 重新汇总成 output.aois"""json_files = sorted(glob.glob(os.path.join(json_dir, "*.json"))) key_frames = [] ghost_active = False # [新增] 标记是否处于消失状态for jf in json_files: with open(jf, "r", encoding="utf-8") as f: data = json.load(f) frame_num = self.extract_frame_number(jf)if frame_num is None or not data.get("shapes"):continueshapes = data["shapes"]seconds = frame_num / (fps or 30)vertices = [{"X": p[0], "Y": p[1]} for p in shapes[0]["points"]]is_ghost = any( sh.get("flags", {}).get("is_ghost", False) for sh in shapes)if is_ghost:if not ghost_active:# 第一次消失 -> 记录到 aoiskey_frames.append({"IsActive": not is_ghost,"Seconds": round(seconds, 6),"Vertices": vertices})ghost_active = Trueelse:# 已经处于消失状态 -> 不记录continueelse:# 目标重新出现 -> 正常记录并重置状态 key_frames.append({"IsActive": not is_ghost,"Seconds": round(seconds, 6),"Vertices": vertices})ghost_active = Falseaois_file = {"Version": 2,"Tags": [],"Media": {"MediaType": 1,"Height": image_height,"Width": image_width,"MediaCount": 1,"DurationMicroseconds": total_us or 0},"Aois": [{"Name": "target","Red": 212, "Green": 0, "Blue": 255,"Tags": [],"KeyFrames": key_frames}]}aois_path = os.path.join(json_dir, "output.aois")with open(aois_path, "w", encoding="utf-8") as af:json.dump(aois_file, af, indent=2, ensure_ascii=False)return aois_path        

初始化的时候加上 self.ghost_active：

class LabelFile:suffix = ".json"def __init__(self, filename=None, image_dir=None):self.shapes = []self.image_path = Noneself.image_data = Noneself.image_dir = image_dirif filename is not None:self.load(filename)self.filename = filenameself.ghost_active = False  # 新增

三、修正save函数

        前面，我们已经新增了按钮，然后修改了json转.aois过程中的问题，即目标消失的时候只在消失的那一帧写入.aois，之后在目标没有出现之前是不写入.aois，这样.aois的大小就不会太大。但是我们跑SAM2视频跟踪的时候它既会生成json也会生成.aois，我们还没改这里的逻辑。

        在anylabeling\views\labeling\label_file.py中，也是同样的新增一个逻辑就是只在消失的那一帧写入.aois。

class LabelFile:suffix = ".json"def __init__(self, filename=None, image_dir=None):self.shapes = []self.image_path = Noneself.image_data = Noneself.image_dir = image_dirif filename is not None:self.load(filename)self.filename = filenameself.ghost_active = Falseself._ghost_active = Falsedef save(self,filename=None,shapes=None,image_path=None,image_height=None,image_width=None,image_data=None,other_data=None,flags=None,total_frames=None,fps=Nonone,total_us=None,):if image_data is not None:image_data = base64.b64encode(image_data).decode("utf-8")image_height, image_width = self._check_image_height_and_width(image_data, image_height, image_width)if other_data is None:other__data = {}if flags is None:flags = {}is_active = Truenum_active = 0for i, shape in enumerate(shapes):if shape["shape_type"] == "rectangle" or shape["shape_type"] == "polygon" or shape["shape_type"] == "rotation":sorted_box = LabelConverter.calculate_bounding_box(shape["points"])xmin, ymin, xmax, ymax = sorted_boxshape["points"] = [[xmin, ymmin],[xmax, ymin],[xmax, ymax],[xmin, ymax],]shapes[i] = shapeif shape.get("flags", {}).get("is_ghost") is True:print("是 ghost")num_active += 1if num_active > 0:is_active = Falsedata = {"version": __version__,"flags": flags,"shapes": shapes,"imagePath": image_path,"imageData": image_data,"imageHeight": image_height,"imageWidth": image_width,}for key, value in other_data.items():assert key not in datadata[key] = valuetry:with utils.io_open(filename, "w") as f:json.dump(data, f, ensure_ascii=False, indent=2)logger.debug(f"[label_file.py save] json.dump() filename:{filename} shapes:{shapes}")self.filename = filenameexcept Exception as e:  # noqaraise LabelFileError(e) from e# 写入aois文件frame_num = self.extract_frame_number(filename)if frame_num is not None and shapes:print(f"[label_file.py][save] frame_num:{frame_num}")print(f"[label_file.py][save] fps:{fps}")seconds = frame_num / fpsvertices = [{'X': p[0], 'Y': p[1]} for p in shapes[0]['points']]print(f"[label_file.py][save] num_active:{num_active}")print(f"[label_file.py][save] is_active:{is_active}")# [新增逻辑] 控制 ghost 只写入一次if not is_active:  # 有 ghostif not self._ghost_active:# 第一次消失 -> 写入 AOIsAOIS_KEY_FRAMES.append({"IsActive": is_active,"Seconds": round(seconds, 6),"Vertices": vertices})self._ghost_active = Trueelse:# 已经处于消失状态 -> 不写入passelse:# 目标重新出现 -> 正常写入并重置 ghost 状态AOIS_KEY_FRAMES.append({"IsActive": is_active,"Seconds": round(seconds, 6),"Vertices": vertices})self._ghost_active = aois_file = {"Version": 2,"Tags": [],"Media": {"MediaType": 1,"Height": image_height,"Width": image_width,"MediaCount": 1,# "DurationMicroseconds": int(total_frames / fps * 1e6)"DurationMicroseconds": total_us},"Aois": [{"Name": shapes[0]..get('label', 'target'),"Red": 212,"Green": 0,"Blue": 255,"Tags": [],"KeyFrames": AOIS_KEY_FRAMES}]}aois_path = os.path.join(os.path.dirname(filename), "output.aois")with open(aois_path, "w", encoding="utf-8") as af:json.dump(aois_file, af, indent=2, ensure_ascii=False)print(f"AOIs 已更新: {aois_path}")

四、修改SAM2视频跟踪后处理（可选）

        前面我们修改了json转.aois的逻辑，但是实际上目标消失的时候json的写入逻辑是没有改的。这个改与不改都没关系，因为最后json转.aois的逻辑改了之后我们就能用.aois导入了，json只是一个中间文件。但是考虑到json也占内存，也可以改成只在消失的那一帧写入json，之后消失那段时间的json是空的。

        在anylabeling/services/auto_labeling/segment_anything_2_video.py中，

    def post_process(self, masks, index=None):"""Post-process the masks produced by the model.Args:masks (np.array): The masks to post-process.index (int, optional): The index of the mask. Defaults to None.Returns:list: A list of Shape objects representing the masks."""# Convert masks to binary formatmasks[masks > 0.0] = 255masks[masks <= 0.0] = 0masks = masks.astype(np.uint8)# Find contours of the maskscontours, _ = cv2.findContours(masks, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)# [修改] contours为空 -> 目标消失if not contours:if self.keep_ghost and obj_id in self._last_valid_boxes:# 只在第一次消失时返回残影if not self._ghost_active.get(obj_id, False):logger.debug(f"[Ghost] obj {obj_id} disappeared, return cached once")ghost_shapes = [s.copy() for s in self._last_valid_boxes[obj_id]]for shape in ghost_shapes:shape.flags["is_ghost"] = True# 标记进入 ghost 状态self._ghost_active[obj_id] = Truereturn ghost_shapesreturn []# Refine and filter contoursapprox_contours = []for contour in contours:# Approximate contour using configurable epsilonepsilon = self.epsilon * cv2.arcLength(contour, True)approx = cv2.approxPolyDP(contour, epsilon, True)[新增] 如果小于某个面积，则过滤掉，即认为目标不存在area = cv2.contourArea(approx)logger.debug(f"[Debug] area: {area}")if area < self.min_area_pixel:logger.debug(f"[Debug] area: {obj_id} < {self.min_area_pixel}")continueapprox_contours.append(approx)# [修改] 面积过滤后为空 -> 目标消失if not approx_contours:if self.keep_ghost and obj_id in self._last_valid_boxes:if not self._ghost_active.get(obj_id, False):logger.debug(f"[Ghost] obj {obj_id} below area, return cached once")ghost_shapes = [s.copy() for s in self._last_valid_boxes[obj_id]]for shape in ghost_shapes:shape.flags["is_ghost"] = Trueself._ghost_active[obj_id] = Truereturn ghost_shapesreturn []# [新增] 有有效轮廓 -> 目标重新出现，重置 ghost 状态self._ghost_active[obj_id] = Falseif len(approx_contours) < 1:return []# Convert contours to shapesshapes = []if self.output_mode == "polygon":for approx in approx_contours:# Scale pointspoints = approx.reshape(-1, 2)points[:, 0] = points[:, 0]points[:, 1] = points[:, 1]points = points.tolist()if len(points) < 3:continuepoints.append(points[0])shape = Shape(flags={})for point in points:point[0] = int(point[0])point[1] = int(point[1])shape.add_point(QtCore.QPointF(point[0], point[1]))# Create Polygon shapeshape.shape_type = "polygon"shape.group_id = (self.group_ids[index] if index is not None else None)shape.closed = Trueshape.label = ("AUTOLABEL_OBJECT" if index is None else self.labels[index])shape.selected = Falseshapes.append(shape)elif self.output_mode == "rectangle":x_min = 100000000y_min = 100000000x_max = 0y_max = 0for approx in approx_contours:points = approx.reshape(-1, 2)points[:, 0] = points[:, 0]points[:, 1] = points[:, 1]points = points.tolist()if len(points) < 3:continuefor point in points:x_min = min(x_min, point[0])y_min = min(y_min, point[1])x_max = max(x_max, point[0])y_max = max(y_max, point[1])shape = Shape(flags={})shape.add_point(QtCore.QPointF(x_min, y_min))shape.add_point(QtCore.QPointF(x_max, y_min))shape.add_point(QtCore.QPointF(x_max, y_max))shape.add_point(QtCore.QPointF(x_min, y_max))shape.shape_type = "rectangle"shape.closed = Trueshape.group_id = (self.group_ids[index] if index is not None else None)shape.fill_color = "#000000"shape.line_color = "#000000"shape.label = ("AUTOLABEL_OBJECT" if index is None else self.labels[index])shape.selected = Falseshapes.append(shape)elif self.output_mode == "rotation":shape = Shape(flags={})rotation_box = get_bounding_boxes(approx_contours[0])[1]for point in rotation_box:shape.add_point(QtCore.QPointF(int(point[0]), int(point[1])))shape.direction = calculate_rotation_theta(rotation_box)shape.shape_type = self.output_modeshape.closed = Trueshape.fill_color = "#000000"shape.line_color = "#000000"shape.label = ("AUTOLABEL_OBJECT" if index is None else self.labels[index])shape.selected = Falseshapes.append(shape)if self.keep_ghost:self._last_valid_boxes[obj_id] = shapesreturn shapes

class SegmentAnything2Video(Model):def __init__(self, config_path, on_message) -> None:self.marks = []self.labels = []self.group_ids = []self.prompts = []self.replace = Trueself.epsilon = 0.001self.min_area_pixel = int(self.config.get("min_area_pixel", 0))   self.keep_ghost = bool(self.config.get("keep_ghost    self._last_valid_boxes = {}  # obj_id -> last Shape list# 初始化时增加一个字典来记录ghost状态self._ghost_active