一刀砍干净 HistoryManager 所有 O(N) stat 全扫路径
继上次只修 hot path 之后, 把残留的所有冷热路径都改成 raw json 直接操作, 彻底消灭 load_history_index 内的 stat 循环. 修改: 1. _migrate_paths_once: 启动时一次性路径归一化, 抽样 5 条 60% 阈值 决定是否需要全量迁移, 之后所有路径都是当前 base_path 下的有效绝对路径 2. load_history_index: 简化为 raw read + from_dict + sort, 0 stat N=513 时只需几 ms (旧版 ~60ms) 3. delete_history_item: 改 raw json filter 4. _cleanup_old_records: 改 raw json sort + slice 5. 删除已无调用方的 _save_history_index / _fix_history_path 至此 hot path (生成完成, 点击历史项) 与 cold path (删除, 启动清理) 都不再触发 N 次 stat. 唯一保留全量读的 load_history_index 也只有 refresh_history 一处真实调用方.
Showing
1 changed file
with
140 additions
and
89 deletions
| ... | @@ -812,6 +812,13 @@ class HistoryManager: | ... | @@ -812,6 +812,13 @@ class HistoryManager: |
| 812 | 812 | ||
| 813 | self.logger.debug(f"历史记录管理器初始化完成,存储路径: {self.base_path}") | 813 | self.logger.debug(f"历史记录管理器初始化完成,存储路径: {self.base_path}") |
| 814 | 814 | ||
| 815 | # 启动时一次性把过期绝对路径归一化到当前 base_path, | ||
| 816 | # 之后 load_history_index 不再做任何 stat 循环 | ||
| 817 | try: | ||
| 818 | self._migrate_paths_once() | ||
| 819 | except Exception as e: | ||
| 820 | self.logger.warning(f"启动路径迁移失败 (可忽略): {e}") | ||
| 821 | |||
| 815 | def save_generation(self, image_bytes: bytes, prompt: str, reference_images: List[bytes], | 822 | def save_generation(self, image_bytes: bytes, prompt: str, reference_images: List[bytes], |
| 816 | aspect_ratio: str, image_size: str, model: str) -> str: | 823 | aspect_ratio: str, image_size: str, model: str) -> str: |
| 817 | """保存生成的图片到历史记录 | 824 | """保存生成的图片到历史记录 |
| ... | @@ -929,80 +936,106 @@ class HistoryManager: | ... | @@ -929,80 +936,106 @@ class HistoryManager: |
| 929 | pass | 936 | pass |
| 930 | return None | 937 | return None |
| 931 | 938 | ||
| 932 | def _fix_history_path(self, stored_path: Path, timestamp: str) -> Path: | 939 | def _migrate_paths_once(self): |
| 933 | """修正历史记录中的路径,使其指向当前 base_path | 940 | """启动时一次性路径归一化(取代过去每次 load 都修正的循环)。 |
| 934 | 941 | ||
| 935 | 存储迁移后,index 里的绝对路径可能指向旧位置(如 .app 内部), | 942 | 历史背景:旧版本 index.json 存绝对路径。.app 重打包或存储位置变更 |
| 936 | 而实际文件已在新的 base_path 下。用 timestamp + 文件名重建路径。 | 943 | 后,老路径失效。过去做法是 load_history_index 每次都全扫 stat 修正, |
| 944 | N=513 时主线程阻塞 ~60ms × N 次/会话,Mac 上累计触发 jetsam SIGKILL。 | ||
| 945 | |||
| 946 | 现在改成只在启动时跑一次: | ||
| 947 | 1. 抽样前 5 条,60% 以上路径有效 → 跳过(绝大多数启动走这条 fast path) | ||
| 948 | 2. 否则全量重写所有路径并 save,之后 load_history_index 直接信任 raw | ||
| 937 | """ | 949 | """ |
| 938 | if stored_path.exists(): | 950 | if not self.history_index_file.exists(): |
| 939 | return stored_path | 951 | return |
| 952 | try: | ||
| 953 | with open(self.history_index_file, 'r', encoding='utf-8') as f: | ||
| 954 | raw = json.load(f) | ||
| 955 | except Exception as e: | ||
| 956 | self.logger.warning(f"_migrate_paths_once 读取 index 失败: {e}") | ||
| 957 | return | ||
| 958 | if not isinstance(raw, list) or not raw: | ||
| 959 | return | ||
| 940 | 960 | ||
| 941 | # 尝试在当前 base_path 下找到对应文件 | 961 | sample = [d for d in raw[:5] if isinstance(d, dict)] |
| 942 | corrected = self.base_path / timestamp / stored_path.name | 962 | if not sample: |
| 943 | if corrected.exists(): | 963 | return |
| 944 | return corrected | 964 | ok = 0 |
| 965 | for d in sample: | ||
| 966 | p = d.get('generated_image_path') | ||
| 967 | if p and Path(p).exists(): | ||
| 968 | ok += 1 | ||
| 969 | # 60% 阈值:少数损坏可能是孤立文件被人手动删,不需要全迁移 | ||
| 970 | if ok * 10 >= len(sample) * 6: | ||
| 971 | self.logger.info( | ||
| 972 | f"[migrate_paths] 抽样 {ok}/{len(sample)} 路径有效,跳过迁移" | ||
| 973 | ) | ||
| 974 | return | ||
| 945 | 975 | ||
| 946 | return stored_path | 976 | self.logger.info( |
| 977 | f"[migrate_paths] 抽样 {ok}/{len(sample)} 路径有效,开始全量迁移 {len(raw)} 条" | ||
| 978 | ) | ||
| 979 | changed = 0 | ||
| 980 | for d in raw: | ||
| 981 | if not isinstance(d, dict): | ||
| 982 | continue | ||
| 983 | ts = d.get('timestamp') | ||
| 984 | if not ts: | ||
| 985 | continue | ||
| 986 | gen_p = d.get('generated_image_path') | ||
| 987 | if gen_p and not Path(gen_p).exists(): | ||
| 988 | cand = self.base_path / ts / Path(gen_p).name | ||
| 989 | if cand.exists(): | ||
| 990 | d['generated_image_path'] = str(cand) | ||
| 991 | changed += 1 | ||
| 992 | refs = d.get('reference_image_paths') or [] | ||
| 993 | new_refs = [] | ||
| 994 | for r in refs: | ||
| 995 | if Path(r).exists(): | ||
| 996 | new_refs.append(r) | ||
| 997 | else: | ||
| 998 | cand = self.base_path / ts / Path(r).name | ||
| 999 | if cand.exists(): | ||
| 1000 | new_refs.append(str(cand)) | ||
| 1001 | changed += 1 | ||
| 1002 | else: | ||
| 1003 | new_refs.append(r) | ||
| 1004 | d['reference_image_paths'] = new_refs | ||
| 1005 | |||
| 1006 | if changed > 0: | ||
| 1007 | self.logger.info(f"[migrate_paths] 修正 {changed} 个路径, 写回 index.json") | ||
| 1008 | try: | ||
| 1009 | with open(self.history_index_file, 'w', encoding='utf-8') as f: | ||
| 1010 | json.dump(raw, f, ensure_ascii=False, indent=2) | ||
| 1011 | except Exception as e: | ||
| 1012 | self.logger.error(f"[migrate_paths] 写回失败: {e}") | ||
| 947 | 1013 | ||
| 948 | def load_history_index(self) -> List[HistoryItem]: | 1014 | def load_history_index(self) -> List[HistoryItem]: |
| 949 | """加载历史记录索引 | 1015 | """加载历史记录索引(仅 raw read + from_dict + sort)。 |
| 950 | 1016 | ||
| 951 | Returns: | 1017 | 路径修正已下放到 __init__ 时的 _migrate_paths_once, |
| 952 | 历史记录项列表,按时间戳倒序排列 | 1018 | load 路径完全没有 stat 系统调用,N=513 时只需几 ms。 |
| 953 | """ | 1019 | """ |
| 954 | if not self.history_index_file.exists(): | 1020 | if not self.history_index_file.exists(): |
| 955 | return [] | 1021 | return [] |
| 956 | |||
| 957 | try: | 1022 | try: |
| 958 | self.logger.info("[load_history_index] 开始读取 index.json") | ||
| 959 | _flush_logs() | ||
| 960 | with open(self.history_index_file, 'r', encoding='utf-8') as f: | 1023 | with open(self.history_index_file, 'r', encoding='utf-8') as f: |
| 961 | data = json.load(f) | 1024 | data = json.load(f) |
| 962 | self.logger.info(f"[load_history_index] JSON 解析完成, {len(data)} 条原始数据") | 1025 | if not isinstance(data, list): |
| 963 | _flush_logs() | 1026 | return [] |
| 964 | 1027 | items: List[HistoryItem] = [] | |
| 965 | history_items = [HistoryItem.from_dict(item) for item in data] | 1028 | for d in data: |
| 966 | total = len(history_items) | 1029 | if not isinstance(d, dict): |
| 967 | 1030 | continue | |
| 968 | # 修正可能过期的绝对路径(存储迁移后旧路径不再有效) | 1031 | try: |
| 969 | # 每条 item 对每个 path 都要 stat; 这是 UI 主线程上可能拖死的点 | 1032 | items.append(HistoryItem.from_dict(d)) |
| 970 | needs_save = False | 1033 | except Exception: |
| 971 | for idx, item in enumerate(history_items): | 1034 | continue |
| 972 | fixed_gen = self._fix_history_path(item.generated_image_path, item.timestamp) | 1035 | items.sort(key=lambda x: x.timestamp, reverse=True) |
| 973 | if fixed_gen != item.generated_image_path: | 1036 | return items |
| 974 | item.generated_image_path = fixed_gen | ||
| 975 | needs_save = True | ||
| 976 | |||
| 977 | fixed_refs = [] | ||
| 978 | for ref_path in item.reference_image_paths: | ||
| 979 | fixed_ref = self._fix_history_path(ref_path, item.timestamp) | ||
| 980 | if fixed_ref != ref_path: | ||
| 981 | needs_save = True | ||
| 982 | fixed_refs.append(fixed_ref) | ||
| 983 | item.reference_image_paths = fixed_refs | ||
| 984 | |||
| 985 | if (idx + 1) % 20 == 0: | ||
| 986 | self.logger.info(f"[load_history_index] 路径修正进度 {idx + 1}/{total}") | ||
| 987 | _flush_logs() | ||
| 988 | |||
| 989 | self.logger.info(f"[load_history_index] 路径修正完成, needs_save={needs_save}") | ||
| 990 | _flush_logs() | ||
| 991 | |||
| 992 | # 路径修正后持久化,避免每次都修正 | ||
| 993 | if needs_save: | ||
| 994 | self.logger.info("检测到历史记录路径变更,已自动修正") | ||
| 995 | self._save_history_index(history_items) | ||
| 996 | _flush_logs() | ||
| 997 | |||
| 998 | # 按时间戳倒序排列 | ||
| 999 | history_items.sort(key=lambda x: x.timestamp, reverse=True) | ||
| 1000 | self.logger.info(f"[load_history_index] 返回 {total} 条") | ||
| 1001 | _flush_logs() | ||
| 1002 | return history_items | ||
| 1003 | except Exception as e: | 1037 | except Exception as e: |
| 1004 | self.logger.error(f"加载历史记录索引失败: {e}", exc_info=True) | 1038 | self.logger.error(f"加载历史记录索引失败: {e}", exc_info=True) |
| 1005 | _flush_logs() | ||
| 1006 | return [] | 1039 | return [] |
| 1007 | 1040 | ||
| 1008 | def get_history_item(self, timestamp: str) -> Optional[HistoryItem]: | 1041 | def get_history_item(self, timestamp: str) -> Optional[HistoryItem]: |
| ... | @@ -1066,14 +1099,26 @@ class HistoryManager: | ... | @@ -1066,14 +1099,26 @@ class HistoryManager: |
| 1066 | if record_dir.exists(): | 1099 | if record_dir.exists(): |
| 1067 | shutil.rmtree(record_dir) | 1100 | shutil.rmtree(record_dir) |
| 1068 | 1101 | ||
| 1069 | # 更新索引文件 | 1102 | # 直接对 raw json 操作, 避免 load_history_index O(N) 全扫 |
| 1070 | history_items = self.load_history_index() | 1103 | if not self.history_index_file.exists(): |
| 1071 | history_items = [item for item in history_items if item.timestamp != timestamp] | 1104 | return True |
| 1072 | self._save_history_index(history_items) | 1105 | try: |
| 1106 | with open(self.history_index_file, 'r', encoding='utf-8') as f: | ||
| 1107 | raw = json.load(f) | ||
| 1108 | if not isinstance(raw, list): | ||
| 1109 | raw = [] | ||
| 1110 | except Exception: | ||
| 1111 | raw = [] | ||
| 1112 | raw = [d for d in raw if isinstance(d, dict) and d.get('timestamp') != timestamp] | ||
| 1113 | try: | ||
| 1114 | with open(self.history_index_file, 'w', encoding='utf-8') as f: | ||
| 1115 | json.dump(raw, f, ensure_ascii=False, indent=2) | ||
| 1116 | except Exception as e: | ||
| 1117 | self.logger.error(f"删除后写回索引失败: {e}") | ||
| 1073 | 1118 | ||
| 1074 | return True | 1119 | return True |
| 1075 | except Exception as e: | 1120 | except Exception as e: |
| 1076 | print(f"删除历史记录失败: {e}") | 1121 | self.logger.error(f"删除历史记录失败: {e}") |
| 1077 | return False | 1122 | return False |
| 1078 | 1123 | ||
| 1079 | def _update_history_index(self, history_item: HistoryItem): | 1124 | def _update_history_index(self, history_item: HistoryItem): |
| ... | @@ -1107,40 +1152,46 @@ class HistoryManager: | ... | @@ -1107,40 +1152,46 @@ class HistoryManager: |
| 1107 | except Exception as e: | 1152 | except Exception as e: |
| 1108 | self.logger.error(f"_update_history_index 写入索引失败: {e}") | 1153 | self.logger.error(f"_update_history_index 写入索引失败: {e}") |
| 1109 | 1154 | ||
| 1110 | def _save_history_index(self, history_items: List[HistoryItem]): | ||
| 1111 | """保存历史记录索引到文件 | ||
| 1112 | |||
| 1113 | Args: | ||
| 1114 | history_items: 历史记录项列表 | ||
| 1115 | """ | ||
| 1116 | try: | ||
| 1117 | data = [item.to_dict() for item in history_items] | ||
| 1118 | with open(self.history_index_file, 'w', encoding='utf-8') as f: | ||
| 1119 | json.dump(data, f, ensure_ascii=False, indent=2) | ||
| 1120 | except Exception as e: | ||
| 1121 | print(f"保存历史记录索引失败: {e}") | ||
| 1122 | |||
| 1123 | def _cleanup_old_records(self): | 1155 | def _cleanup_old_records(self): |
| 1124 | """清理旧的历史记录,保持最大数量限制。max_history_count <= 0 表示不限制。""" | 1156 | """清理旧的历史记录,保持最大数量限制。max_history_count <= 0 表示不限制。""" |
| 1125 | if self.max_history_count <= 0: | 1157 | if self.max_history_count <= 0: |
| 1126 | return | 1158 | return |
| 1127 | history_items = self.load_history_index() | 1159 | if not self.history_index_file.exists(): |
| 1128 | if len(history_items) > self.max_history_count: | 1160 | return |
| 1129 | # 保留最新的记录 | 1161 | try: |
| 1130 | items_to_keep = history_items[:self.max_history_count] | 1162 | with open(self.history_index_file, 'r', encoding='utf-8') as f: |
| 1131 | items_to_remove = history_items[self.max_history_count:] | 1163 | raw = json.load(f) |
| 1132 | 1164 | if not isinstance(raw, list): | |
| 1133 | # 删除多余记录的文件 | 1165 | return |
| 1134 | for item in items_to_remove: | 1166 | except Exception as e: |
| 1135 | record_dir = self.base_path / item.timestamp | 1167 | self.logger.error(f"_cleanup_old_records 读取索引失败: {e}") |
| 1168 | return | ||
| 1169 | |||
| 1170 | # 倒序排列,保留前 N 条 | ||
| 1171 | raw.sort(key=lambda d: d.get('timestamp', '') if isinstance(d, dict) else '', reverse=True) | ||
| 1172 | if len(raw) <= self.max_history_count: | ||
| 1173 | return | ||
| 1174 | keep = raw[:self.max_history_count] | ||
| 1175 | remove = raw[self.max_history_count:] | ||
| 1176 | |||
| 1177 | for d in remove: | ||
| 1178 | if not isinstance(d, dict): | ||
| 1179 | continue | ||
| 1180 | ts = d.get('timestamp') | ||
| 1181 | if not ts: | ||
| 1182 | continue | ||
| 1183 | record_dir = self.base_path / ts | ||
| 1136 | if record_dir.exists(): | 1184 | if record_dir.exists(): |
| 1137 | try: | 1185 | try: |
| 1138 | shutil.rmtree(record_dir) | 1186 | shutil.rmtree(record_dir) |
| 1139 | except Exception as e: | 1187 | except Exception as e: |
| 1140 | print(f"删除旧历史记录失败 {item.timestamp}: {e}") | 1188 | self.logger.warning(f"删除旧记录失败 {ts}: {e}") |
| 1141 | 1189 | ||
| 1142 | # 更新索引文件 | 1190 | try: |
| 1143 | self._save_history_index(items_to_keep) | 1191 | with open(self.history_index_file, 'w', encoding='utf-8') as f: |
| 1192 | json.dump(keep, f, ensure_ascii=False, indent=2) | ||
| 1193 | except Exception as e: | ||
| 1194 | self.logger.error(f"_cleanup_old_records 写回索引失败: {e}") | ||
| 1144 | 1195 | ||
| 1145 | 1196 | ||
| 1146 | class LoginDialog(QDialog): | 1197 | class LoginDialog(QDialog): | ... | ... |
-
Please register or sign in to post a comment