Pending changes exported from your codespace

2026-10-17 05:20:46 +08:00 · 2023-06-13 08:38:47 +00:00
parent 2911be68ca
commit d7858064ec
3 changed files with 131 additions and 108 deletions
@@ -1,71 +0,0 @@
-from pathlib import Path
-import os
-
-current_path = Path("./")
-
-_ = lambda x, y: os.environ.get(x, y) or y
-__ = lambda x, y: _(x, str(y)).lower() == str(y).lower()
-
-
-class Config:
-    """
-    Default configuration , Can be effetively changed by env variables.
-    DEMO:
-        export DEBUG=FALSE
-        export MYSQL_HOST=192.168.1.1
-    """
-
-    debug = __("DEBUG", True)
-    domains = (
-        _(
-            "DOMAIN",
-            "xxxxxxxx3a3kuuhhw5w7stk25fzhttrlpiomij5bogkg7yyqsng5tqyd.onion",
-        ),
-    )
-
-    mysql_host = _("MYSQL_HOST", "127.0.0.1")
-    mysql_port = int(_("MYSQL_PORT", "3306"))
-    mysql_usr = _("MYSQL_USR", "root")
-    mysql_pass = _("MYSQL_PASS", "root")
-    mysql_db = _("MYSQL_DB", "db")
-
-    tor_proxy = _("TOR_PROXY", "socks5h://127.0.0.1:9050")
-
-    tg_proxy = _("TG_PROXY", None)
-    tg_token = _("TG_TOKEN", None)
-    tg_channel_id_darknet = _("TG_CID_DARKNET", None)
-    tg_report_group_id = _("TG_REPORT_GID", None)
-
-    no_porn_img = __("NO_PORN", False)
-    send_for_test = __("SEND_FOR_TEST", True)
-
-    filter_area = tuple(
-        [
-            item
-            for item in (
-                ("/ea.php?ea=10001", "数据资源"),
-                ("/ea.php?ea=10006", "服务业务"),
-                ("/ea.php?ea=10002", "虚拟物品"),
-                ("/ea.php?ea=10007", "实体物品"),
-                ("/ea.php?ea=10003", "技术技能"),
-                ("/ea.php?ea=10004", "影视色情"),
-                ("/ea.php?ea=10010", "其它类别"),
-                ("/ea.php?ea=10005", "基础知识"),
-                ("/ea.php?ea=10009", "私人专拍"),
-            )
-            if not _("FILTER_AREA", "")
-            or item[0].split("=")[1] in _("FILTER_AREA", "")
-        ]
-    )
-
-    just_update = __("JUST_UPDATE", True)
-    screen_path = Path(_("ABS_SCREEN_PATH", current_path / "screen_shot"))
-    max_log_len = int(_("MAX_LOG_LEN", "120"))
-    limit_log = __("LIMIT_LOG", True)
-    notice_range_days = int(_("NOTICE_RANGE_DAYS", "2"))
-
-
-if __name__ == "__main__":
-    from pprint import pprint
-
-    pprint(Config.__dict__)
@@ -1,51 +1,75 @@
 import datetime

-import pymysql
-from peewee import *
-from peewee import __exception_wrapper__
+
+from peewee import (
+    CharField,
+    DateTimeField,
+    BooleanField,
+    TextField,
+    Model,
+    SqliteDatabase,
+    MySQLDatabase,
+    OperationalError,
+    __exception_wrapper__,
+)
+
+from settings import (
+    MYSQL_DATABASE,
+    MYSQL_HOST,
+    MYSQL_PASSWORD,
+    MYSQL_PORT_NUMBER,
+    MYSQL_USER,
+    USE_MYSQL,
+)

 from darknet.default import Config

-Links = {
-    "host": Config.mysql_host,
-    "port": Config.mysql_port,
-    "user": Config.mysql_usr,
-    "password": Config.mysql_pass,
-}

-try:
-    con = pymysql.connect(**Links)
-    with con.cursor() as cursor:
-        cursor.execute(
-            f"create database {Config.mysql_db} character set UTF8mb4 collate utf8mb4_bin"
-        )
-    con.close()
-except pymysql.err.ProgrammingError as e:
-    if "1007" in str(e):
-        pass
-except Exception as e:
-    raise e
+if USE_MYSQL:
+    import pymysql

+    links = {
+        "host": MYSQL_HOST,
+        "port": MYSQL_PORT_NUMBER,
+        "user": MYSQL_USER,
+        "password": MYSQL_PASSWORD,
+    }

-class RetryOperationalError(object):
-    def execute_sql(self, sql, params=None, commit=True):
-        try:
-            cursor = super(RetryOperationalError, self).execute_sql(
-                sql, params, commit
+    try:
+        con = pymysql.connect(**links)
+        with con.cursor() as cursor:
+            cursor.execute(
+                f"create database {MYSQL_DATABASE} character set UTF8mb4 collate utf8mb4_bin"
            )
-        except OperationalError:
-            if not self.is_closed():
-                self.close()
-            with __exception_wrapper__:
-                cursor = self.cursor()
-                cursor.execute(sql, params or ())
-                if commit and not self.in_transaction():
-                    self.commit()
-        return cursor
+        con.close()
+    except pymysql.err.ProgrammingError as e:
+        if "1007" in str(e):
+            pass
+    except Exception as e:
+        raise e

+    class RetryOperationalError(object):
+        def execute_sql(self, sql, params=None, commit=True):
+            try:
+                cursor = super(RetryOperationalError, self).execute_sql(
+                    sql, params, commit
+                )
+            except OperationalError:
+                if not self.is_closed():
+                    self.close()
+                with __exception_wrapper__:
+                    cursor = self.cursor()
+                    cursor.execute(sql, params or ())
+                    if commit and not self.in_transaction():
+                        self.commit()
+            return cursor

-class RetryMySQLDatabase(RetryOperationalError, MySQLDatabase):
-    pass
+    class RetryMySQLDatabase(RetryOperationalError, MySQLDatabase):
+        pass
+
+    links["database"] = MYSQL_DATABASE
+    db = RetryMySQLDatabase(**links, charset="utf8mb4")
+else:


 Links["database"] = Config.mysql_db
@@ -0,0 +1,70 @@
+
+from os import getenv
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+BASE_FOLDER = Path(__file__).parent.parent
+ENV_FILE_PATH = BASE_FOLDER / ".env"
+SITE_DIR = BASE_FOLDER / "site"
+VOL_FOLDER = BASE_FOLDER / "vol"
+RESOURCE_DIR = VOL_FOLDER / "resource"
+SCREEN_SHOT_PATH = VOL_FOLDER / "screen_shot"
+
+
+load_dotenv(ENV_FILE_PATH)
+
+
+def env_bool(var_name: str, defaults: bool = False) -> bool:
+    """
+    env boolean check
+    """
+    return str(getenv(var_name, str(defaults))).lower() == "True".lower()
+
+
+DEBUG = env_bool("DEBUG", True)
+
+# mysql database
+USE_MYSQL = env_bool("USE_MYSQL")
+MYSQL_DATABASE = getenv("MYSQL_DATABASE", "nlp")
+MYSQL_USER = getenv("MYSQL_USER", "root")
+MYSQL_PASSWORD = getenv("MYSQL_PASSWORD", "")
+MYSQL_HOST = getenv("MYSQL_HOST", "127.0.0.1")
+MYSQL_PORT_NUMBER = int(getenv("MYSQL_PORT_NUMBER", "3306"))
+
+# sqlite database path
+DATABASE_PATH = getenv("DATABASE_PATH", VOL_FOLDER / "data.db")
+
+
+# tor config
+TOR_PROXY = getenv("TOR_PROXY", "socks5h://localhost:9050")
+
+# tg config
+TG_TOKEN = getenv("TG_TOKEN", None)
+TG_CHANNEL_ID_DARK_NET = getenv("TG_CHANNEL_ID_DARK_NET", None)
+TG_REPORT_GROUP_ID = getenv("TG_REPORT_GROUP_ID", None)
+TG_PROXY = getenv("TG_PROXY", None)
+
+# porn image filter
+FILTER_PORN_IMAGE = env_bool("FILTER_PORN_IMAGE", False)
+
+# send for test
+SEND_FOR_TEST = env_bool("SEND_FOR_TEST", True)
+
+# log config
+LOG_LEVEL = getenv("LOG_LEVEL", "INFO")
+LOG_FILE = getenv("LOG_FILE", VOL_FOLDER / "log.log")
+LOG_FORMAT = getenv("LOG_FORMAT", "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+# redis config
+USE_REDIS = env_bool("USE_REDIS")
+REDIS_HOST = getenv("REDIS_HOST", "127.0.0.1")
+REDIS_PORT = int(getenv("REDIS_PORT_NUMBER", "6379"))
+REDIS_PASSWORD = getenv("REDIS_PASSWORD", "")
+REDIS_DB = getenv("REDIS_DB", "0")
+REDIS_URI = getenv(
+    "REDIS_URI",
+    f"redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}",
+)
+# sqlite cache
+CELERY_CACHE_PATH = getenv("CELERY_CACHE_PATH", VOL_FOLDER / "cache.db")