From efba8c18bdc91ddae2df89baffd80415bbd5b418 Mon Sep 17 00:00:00 2001 From: Infstellar <2644641825@qq.com> Date: Thu, 27 Feb 2025 21:03:12 +0800 Subject: [PATCH] some optimizations about config loading --- config/settings.py | 50 ++++++++++++++++++++++++++--------- pages/label_images.py | 8 +++--- services/embedding_service.py | 1 + services/image_search.py | 7 +++++ 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/config/settings.py b/config/settings.py index 227749a..46726bb 100644 --- a/config/settings.py +++ b/config/settings.py @@ -8,6 +8,19 @@ import typing as t from config.utils import * + +""" +修改配置：用with打开Config，修改后自动保存。可以修改多个配置。 +with Config() as config: + config.api.embedding_models.base_url = '123' +使用配置：每次使用前实例化Config +！！！注意：如果不重新实例化，config的值不会改变！！！ +应当将Config视为一次性的实例。如果要缓存这个实例，应当保证这个缓存在使用的生命周期内，config不会被改变。 +对于改不了的屎山代码，请在实例化Config时，将keep_tracked参数设置为True。注意：设置后，这个缓存将是只读的。 + +每次重新实例化不会重新读盘。在内存中有一个缓存。 +""" + CONFIG_DIR = os.path.dirname(os.path.abspath(__file__)) CONFIG_FILE = os.path.join(CONFIG_DIR, 'config.yaml') CONFIG_EXAMPLE_FILE = os.path.join(CONFIG_DIR, 'config.example.yaml') @@ -71,17 +84,17 @@ class BaseConfig(BaseModel, frozen=False, extra='allow'): # 'value': value # }) -class ObserverdDict(dict): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.__dict__['settled_dicts'] = [] - def __setitem__(self, key, value): - print(f"setitem:{key}-{value}") - if not isinstance(value, BaseConfig): - self.__dict__['settled_dicts'].append({ - 'key': [key], - 'value': value - }) +# class ObserverdDict(dict): +# def __init__(self, *args, **kwargs): +# super().__init__(*args, **kwargs) +# self.__dict__['settled_dicts'] = [] +# def __setitem__(self, key, value): +# print(f"setitem:{key}-{value}") +# if not isinstance(value, BaseConfig): +# self.__dict__['settled_dicts'].append({ +# 'key': [key], +# 'value': value +# }) class EmbeddingModelConfig(BaseConfig): name: str @@ -179,7 +192,7 @@ class Config(BaseConfig): - def __init__(self): + def __init__(self, keep_tracked = False): if sys.gettrace() is not None: frame = inspect.currentframe().f_back filename = frame.f_code.co_filename @@ -191,9 +204,13 @@ class Config(BaseConfig): print('loading config') data = load_yaml_file(CONFIG_FILE) config_cache.set_config(data) + if keep_tracked: + self.__dict__['keep_tracked'] = True super().__init__(**data) def __enter__(self): + if self.__dict__['keep_tracked']: + raise RuntimeError('Config object is being tracked, which cannot be modified.') return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -213,7 +230,6 @@ class Config(BaseConfig): # print('Config object is being deleted') - @property def base_dir(self) -> str: """获取项目根目录""" @@ -250,6 +266,12 @@ class Config(BaseConfig): """获取缓存文件的绝对路径""" return os.path.join(self.base_dir,self.paths.label_images_cache_file) + def __getattr__(self, item): + if self.__dict__['keep_tracked']: + return Config().__getattr__(item) + else: + return super().__getattr__(item) + # def reload(self) -> None: # """重新加载配置文件""" # new_config = Config() @@ -267,5 +289,7 @@ if __name__ == '__main__': with Config() as config: config.api.embedding_models.base_url = '123' """使用配置：每次使用前实例化Config""" + """注意：如果不重新实例化，config的值不会改变！应当将Config视为一次新的实例。如果要缓存这个实例，应当保证config缓存在使用的生命周期内不会改变。""" print(Config().api.embedding_models.base_url) + diff --git a/pages/label_images.py b/pages/label_images.py index 8d02024..2da0a81 100644 --- a/pages/label_images.py +++ b/pages/label_images.py @@ -67,8 +67,8 @@ if st.session_state.result_folder_name == '' and 'image_folder_name' in st.sessi st.session_state.result_folder_name = st.session_state.image_folder_name if 'pre_generate_result' not in st.session_state: st.session_state.pre_generate_result = {} -if 'resource_pack_service' not in st.session_state: - st.session_state.resource_pack_service = ResourcePackService() +# if 'resource_pack_service' not in st.session_state: +# st.session_state.resource_pack_service = ResourcePackService() # api if 'api_key' not in st.session_state: @@ -211,7 +211,7 @@ with st.sidebar: cover_path = st.session_state.cropped_cover_path if pack_cover else None # 创建资源包 - pack_dir = st.session_state.resource_pack_service.create_resource_pack( + pack_dir = ResourcePackService().create_resource_pack( name=pack_name, version=pack_version, author=pack_author, @@ -224,7 +224,7 @@ with st.sidebar: # 生成zip文件 try: - zip_path = st.session_state.resource_pack_service.export_resource_pack(pack_dir) + zip_path = ResourcePackService().export_resource_pack(pack_dir) # 提供zip文件下载 with open(zip_path, "rb") as f: diff --git a/services/embedding_service.py b/services/embedding_service.py index d1b093b..d05d31f 100644 --- a/services/embedding_service.py +++ b/services/embedding_service.py @@ -18,6 +18,7 @@ import threading class EmbeddingService: def __init__(self): + self.config = Config(keep_tracked=True) self.api_key = Config().api.embedding_models.api_key self.base_url = Config().api.embedding_models.base_url self.local_models = {} diff --git a/services/image_search.py b/services/image_search.py index 49ebcbf..0363387 100644 --- a/services/image_search.py +++ b/services/image_search.py @@ -22,7 +22,11 @@ class ImageSearch: self.image_data = None self._try_load_cache() + def __reload_class_cache(self): + self.embedding_service = EmbeddingService() + def _try_load_cache(self) -> None: + self.__reload_class_cache() """尝试加载缓存""" # 获取所有启用的资源包的缓存文件 cache_files = self.resource_pack_manager.get_cache_files() @@ -133,6 +137,7 @@ class ImageSearch: return self.image_data is not None def generate_cache(self, progress_bar) -> None: + self.__reload_class_cache() """生成缓存""" if self.embedding_service.mode == 'local': # 确保模型已加载 @@ -170,6 +175,7 @@ class ImageSearch: raise RuntimeError(error_message) def _generate_pack_cache(self, pack_id: str, pack_info: Dict, progress_bar) -> None: + self.__reload_class_cache() """为指定的资源包生成缓存""" img_dir = pack_info["path"] if not os.path.isabs(img_dir): @@ -359,6 +365,7 @@ class ImageSearch: return np.dot(a, b) def search(self, query: str, top_k: int = 5, api_key: Optional[str] = None) -> List[str]: + self.__reload_class_cache() """语义搜索最匹配的图片""" if not self.has_cache(): return []