diff --git a/migrate_01_article_fp_rank.py b/migrate_01_article_fp_rank.py index 1d6c234..de161d6 100644 --- a/migrate_01_article_fp_rank.py +++ b/migrate_01_article_fp_rank.py @@ -71,7 +71,7 @@ async def main() -> None: data_to_save: List[ArticleEarningRankingRecordDocument] = [] async for item in OldArticleFPRank.find_many( - sort={"date": "ASC", "ranking": "ASC"} + {}, sort={"date": "ASC", "ranking": "ASC"} ): await insert_or_update_user(item) data_to_save.append(await convert_item(item)) diff --git a/migrate_02_assets_rank.py b/migrate_02_assets_rank.py index 37059df..9f3f2e7 100644 --- a/migrate_02_assets_rank.py +++ b/migrate_02_assets_rank.py @@ -67,7 +67,9 @@ async def main() -> None: ) data_to_save: List[AssetsRankingRecordDocument] = [] - async for item in OldAssetsRank.find_many(sort={"date": "ASC", "ranking": "ASC"}): + async for item in OldAssetsRank.find_many( + {}, sort={"date": "ASC", "ranking": "ASC"} + ): await insert_or_update_user(item) data_to_save.append(await convert_item(item)) diff --git a/migrate_03_daily_update_rank.py b/migrate_03_daily_update_rank.py index 4b7c2e5..5f45573 100644 --- a/migrate_03_daily_update_rank.py +++ b/migrate_03_daily_update_rank.py @@ -60,7 +60,7 @@ async def main() -> None: data_to_save: List[DailyUpdateRankingRecordDocument] = [] async for item in OldDailyUpdateRank.find_many( - sort={"date": "ASC", "ranking": "ASC"} + {}, sort={"date": "ASC", "ranking": "ASC"} ): await insert_or_update_user(item) data_to_save.append(await convert_item(item)) diff --git a/migrate_04_lp_collections.py b/migrate_04_lp_collections.py index a4e658b..a588db4 100644 --- a/migrate_04_lp_collections.py +++ b/migrate_04_lp_collections.py @@ -72,7 +72,7 @@ async def main() -> None: data_to_save: List[LPRecommendedArticleRecordDocument] = [] async for item in OldLPCollections.find_many( - sort={"date": "ASC", "ranking": "ASC"} + {}, sort={"date": "ASC", "ranking": "ASC"} ): await insert_or_update_user(item) data_to_save.append(await convert_item(item)) diff --git a/migrate_05_lottery_data.py b/migrate_05_lottery_data.py index c70bfb8..a7b0e42 100644 --- a/migrate_05_lottery_data.py +++ b/migrate_05_lottery_data.py @@ -52,7 +52,9 @@ async def main() -> None: logger.info(f"旧集合数据量：{old_data_count}") data_to_save: List[LotteryWinRecordDocument] = [] - async for item in OldLotteryData.find_many(sort={"date": "ASC", "ranking": "ASC"}): + async for item in OldLotteryData.find_many( + {}, sort={"date": "ASC", "ranking": "ASC"} + ): await insert_or_update_user(item) data_to_save.append(await convert_item(item)) diff --git a/migrate_06_jpep.py b/migrate_06_jpep.py index 9a2c147..b356d65 100644 --- a/migrate_06_jpep.py +++ b/migrate_06_jpep.py @@ -61,7 +61,7 @@ async def main() -> None: logger.info(f"旧集合数据量：{old_data_count}") data_to_save: List[FTNTradeOrderDocument] = [] - async for item in OldJPEPFTNMacket.find_many(sort={"fetch_time": "ASC"}): + async for item in OldJPEPFTNMacket.find_many({}, sort={"fetch_time": "ASC"}): await insert_or_update_user(item) data_to_save.append(await convert_item(item)) diff --git a/models/jianshu/article_earning_ranking_record.py b/models/jianshu/article_earning_ranking_record.py index d63875b..2b56a1b 100644 --- a/models/jianshu/article_earning_ranking_record.py +++ b/models/jianshu/article_earning_ranking_record.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import ClassVar, List, Optional +from typing import Optional from jkit.msgspec_constraints import ( ArticleSlug, @@ -8,28 +8,22 @@ PositiveInt, UserSlug, ) -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from utils.db import JIANSHU_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class ArticleField(Field, **FIELD_OBJECT_CONFIG): +class ArticleField(Field, **MODEL_CONFIG): slug: Optional[ArticleSlug] title: Optional[NonEmptyStr] -class EarningField(Field, **FIELD_OBJECT_CONFIG): +class EarningField(Field, **MODEL_CONFIG): to_author: PositiveFloat to_voter: PositiveFloat -class ArticleEarningRankingRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class ArticleEarningRankingRecordDocument(Document, **MODEL_CONFIG): date: datetime ranking: PositiveInt @@ -39,6 +33,4 @@ class ArticleEarningRankingRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = JIANSHU_DB.article_earning_ranking_records - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "ranking"], unique=True), - ] + indexes = (Index(keys=("date", "ranking"), unique=True),) diff --git a/models/jianshu/assets_ranking_record.py b/models/jianshu/assets_ranking_record.py index 74a5e20..567e363 100644 --- a/models/jianshu/assets_ranking_record.py +++ b/models/jianshu/assets_ranking_record.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import ClassVar, List, Optional +from typing import Optional from jkit.msgspec_constraints import ( NonNegativeFloat, @@ -7,24 +7,18 @@ PositiveInt, UserSlug, ) -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from utils.db import JIANSHU_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class AmountField(Field, **FIELD_OBJECT_CONFIG): +class AmountField(Field, **MODEL_CONFIG): fp: Optional[NonNegativeFloat] ftn: Optional[NonNegativeFloat] assets: Optional[PositiveFloat] -class AssetsRankingRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class AssetsRankingRecordDocument(Document, **MODEL_CONFIG): date: datetime ranking: PositiveInt @@ -33,6 +27,4 @@ class AssetsRankingRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = JIANSHU_DB.assets_ranking_records - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "ranking"], unique=True), - ] + indexes = (Index(keys=("date", "ranking"), unique=True),) diff --git a/models/jianshu/daily_update_ranking_record.py b/models/jianshu/daily_update_ranking_record.py index b485ce4..3224321 100644 --- a/models/jianshu/daily_update_ranking_record.py +++ b/models/jianshu/daily_update_ranking_record.py @@ -1,20 +1,15 @@ from datetime import datetime -from typing import ClassVar, List from jkit.msgspec_constraints import ( PositiveInt, UserSlug, ) -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Index from utils.db import JIANSHU_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - Document, -) -class DailyUpdateRankingRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class DailyUpdateRankingRecordDocument(Document, **MODEL_CONFIG): date: datetime ranking: PositiveInt days: PositiveInt @@ -23,6 +18,4 @@ class DailyUpdateRankingRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = JIANSHU_DB.daily_update_ranking_records - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "userSlug"], unique=True), - ] + indexes = (Index(keys=("date", "userSlug"), unique=True),) diff --git a/models/jianshu/lottery_win_record.py b/models/jianshu/lottery_win_record.py index ec75a90..2c6fba6 100644 --- a/models/jianshu/lottery_win_record.py +++ b/models/jianshu/lottery_win_record.py @@ -1,21 +1,16 @@ from datetime import datetime -from typing import ClassVar, List from jkit.msgspec_constraints import ( NonEmptyStr, PositiveInt, UserSlug, ) -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Index from utils.db import JIANSHU_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - Document, -) -class LotteryWinRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class LotteryWinRecordDocument(Document, **MODEL_CONFIG): id: PositiveInt time: datetime award_name: NonEmptyStr @@ -24,13 +19,11 @@ class LotteryWinRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = JIANSHU_DB.lottery_win_records - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["id"], unique=True), - ] + indexes = (Index(keys=("id",), unique=True),) @classmethod async def get_latest_record_id(cls) -> int: - latest_data = await LotteryWinRecordDocument.find_one(sort={"id": "DESC"}) + latest_data = await LotteryWinRecordDocument.find_one({}, sort={"id": "DESC"}) if not latest_data: return 0 diff --git a/models/jianshu/lp_recommend_article_record.py b/models/jianshu/lp_recommend_article_record.py index 96421c1..5be9fd5 100644 --- a/models/jianshu/lp_recommend_article_record.py +++ b/models/jianshu/lp_recommend_article_record.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import ClassVar, List from jkit.msgspec_constraints import ( ArticleSlug, @@ -10,16 +9,12 @@ UserSlug, ) from msgspec import field -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Index from utils.db import JIANSHU_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - Document, -) -class LPRecommendedArticleRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class LPRecommendedArticleRecordDocument(Document, **MODEL_CONFIG): date: datetime id: PositiveInt slug: ArticleSlug @@ -40,9 +35,7 @@ class LPRecommendedArticleRecordDocument(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = JIANSHU_DB.lp_recommended_article_records - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "slug"], unique=True), - ] + indexes = (Index(("date", "slug"), unique=True),) @classmethod async def is_record_exist(cls, slug: str) -> bool: diff --git a/models/jianshu/user.py b/models/jianshu/user.py index b8bfd8e..98d8393 100644 --- a/models/jianshu/user.py +++ b/models/jianshu/user.py @@ -1,12 +1,11 @@ from datetime import datetime from enum import Enum -from typing import Any, ClassVar, Dict, List, Optional +from typing import Any, Dict, List, Optional from jkit.msgspec_constraints import PositiveInt, UserName, UserSlug, UserUploadedUrl -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Index from utils.db import JIANSHU_DB -from utils.document_model import Document class JianshuUserStatus(Enum): @@ -14,7 +13,7 @@ class JianshuUserStatus(Enum): INACCESSABLE = "INACCESSIBLE" -class JianshuUserDocument(Document): +class JianshuUserDocument(Document, **MODEL_CONFIG): slug: UserSlug status: JianshuUserStatus updated_at: datetime @@ -25,10 +24,10 @@ class JianshuUserDocument(Document): class Meta: # type: ignore collection = JIANSHU_DB.users - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["slug"], unique=True), - IndexModel(["updatedAt"]), - ] + indexes = ( + Index(keys=("slug",), unique=True), + Index(keys=("updatedAt",)), + ) @classmethod async def is_record_exist(cls, slug: str) -> bool: diff --git a/models/jpep/credit_history.py b/models/jpep/credit_history.py index be64d18..b3c8933 100644 --- a/models/jpep/credit_history.py +++ b/models/jpep/credit_history.py @@ -1,26 +1,20 @@ from datetime import datetime -from typing import ClassVar, List, Optional +from typing import Optional from jkit.msgspec_constraints import NonNegativeInt, PositiveInt -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Index from utils.db import JPEP_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - Document, -) -class CreditHistoryDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class CreditHistoryDocument(Document, **MODEL_CONFIG): time: datetime user_id: PositiveInt value: NonNegativeInt class Meta: # type: ignore collection = JPEP_DB.credit_history - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["time", "userId"], unique=True), - ] + indexes = (Index(keys=("time", "userId"), unique=True),) @classmethod async def get_latest_value(cls, user_id: int) -> Optional[int]: diff --git a/models/jpep/ftn_trade_order.py b/models/jpep/ftn_trade_order.py index 9beb471..49bfa4c 100644 --- a/models/jpep/ftn_trade_order.py +++ b/models/jpep/ftn_trade_order.py @@ -1,30 +1,24 @@ from datetime import datetime -from typing import ClassVar, List, Literal +from typing import Literal from jkit.msgspec_constraints import ( NonNegativeInt, PositiveFloat, PositiveInt, ) -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from utils.db import JPEP_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class AmountField(Field, **FIELD_OBJECT_CONFIG): +class AmountField(Field, **MODEL_CONFIG): total: PositiveInt traded: NonNegativeInt tradable: NonNegativeInt minimum_trade: PositiveInt -class FTNTradeOrderDocument(Document, **DOCUMENT_OBJECT_CONFIG): +class FTNTradeOrderDocument(Document, **MODEL_CONFIG): fetch_time: datetime id: PositiveInt published_at: datetime @@ -37,6 +31,4 @@ class FTNTradeOrderDocument(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = JPEP_DB.ftn_trade_orders - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["fetchTime", "id"], unique=True), - ] + indexes = (Index(keys=("fetchTime", "id"), unique=True),) diff --git a/models/jpep/user.py b/models/jpep/user.py index b312134..775756e 100644 --- a/models/jpep/user.py +++ b/models/jpep/user.py @@ -1,14 +1,13 @@ from datetime import datetime -from typing import Any, ClassVar, Dict, List, Optional +from typing import Any, Dict, Optional from jkit.msgspec_constraints import NonNegativeInt, PositiveInt -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from utils.db import JPEP_DB -from utils.document_model import FIELD_OBJECT_CONFIG, Document, Field -class CreditHistoryFieldItem(Field, **FIELD_OBJECT_CONFIG): +class CreditHistoryFieldItem(Field, **MODEL_CONFIG): time: datetime value: NonNegativeInt @@ -22,10 +21,10 @@ class UserDocument(Document): class Meta: # type: ignore collection = JPEP_DB.users - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["id"], unique=True), - IndexModel(["updatedAt"]), - ] + indexes = ( + Index(keys=("id",), unique=True), + Index(keys=("updatedAt",)), + ) @classmethod async def is_record_exist(cls, id: int) -> bool: # noqa: A002 diff --git a/old_models/article_fp_rank.py b/old_models/article_fp_rank.py index 0d08f70..0207c64 100644 --- a/old_models/article_fp_rank.py +++ b/old_models/article_fp_rank.py @@ -1,36 +1,30 @@ from datetime import datetime -from typing import ClassVar, List, Optional +from typing import Optional from msgspec import field -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from old_models import OLD_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class OldArticleField(Field, **FIELD_OBJECT_CONFIG): +class OldArticleField(Field, **MODEL_CONFIG): title: Optional[str] url: Optional[str] -class OldAuthorField(Field, **FIELD_OBJECT_CONFIG): +class OldAuthorField(Field, **MODEL_CONFIG): name: Optional[str] = None id: Optional[int] = None url: Optional[str] = None -class OldRewardField(Field, **FIELD_OBJECT_CONFIG): +class OldRewardField(Field, **MODEL_CONFIG): to_author: float = field(name="to_author") to_voter: float = field(name="to_voter") total: float -class OldArticleFPRank(Document, **DOCUMENT_OBJECT_CONFIG): +class OldArticleFPRank(Document, **MODEL_CONFIG): date: datetime ranking: int @@ -40,6 +34,4 @@ class OldArticleFPRank(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = OLD_DB.article_FP_rank - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "ranking"], unique=True), - ] + indexes = (Index(keys=("date", "ranking"), unique=True),) diff --git a/old_models/assets_rank.py b/old_models/assets_rank.py index 1f295fc..e72f92a 100644 --- a/old_models/assets_rank.py +++ b/old_models/assets_rank.py @@ -1,31 +1,25 @@ from datetime import datetime -from typing import ClassVar, List, Optional +from typing import Optional from msgspec import field -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from old_models import OLD_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class OldUserField(Field, **FIELD_OBJECT_CONFIG): +class OldUserField(Field, **MODEL_CONFIG): id: Optional[int] url: Optional[str] name: Optional[str] -class OldAssetsField(Field, **FIELD_OBJECT_CONFIG): +class OldAssetsField(Field, **MODEL_CONFIG): fp: Optional[float] = field(name="FP") ftn: Optional[float] = field(name="FTN") total: Optional[float] -class OldAssetsRank(Document, **DOCUMENT_OBJECT_CONFIG): +class OldAssetsRank(Document, **MODEL_CONFIG): date: datetime ranking: int @@ -34,6 +28,4 @@ class OldAssetsRank(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = OLD_DB.assets_rank - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "ranking"], unique=True), - ] + indexes = (Index(keys=("date", "ranking"), unique=True),) diff --git a/old_models/daily_update_rank.py b/old_models/daily_update_rank.py index 6983427..0b11b44 100644 --- a/old_models/daily_update_rank.py +++ b/old_models/daily_update_rank.py @@ -1,23 +1,16 @@ from datetime import datetime -from typing import ClassVar, List -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from old_models import OLD_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class OldUserField(Field, **FIELD_OBJECT_CONFIG): +class OldUserField(Field, **MODEL_CONFIG): url: str name: str -class OldDailyUpdateRank(Document, **DOCUMENT_OBJECT_CONFIG): +class OldDailyUpdateRank(Document, **MODEL_CONFIG): date: datetime ranking: int @@ -26,6 +19,4 @@ class OldDailyUpdateRank(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = OLD_DB.daily_update_rank - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["date", "ranking"]), - ] + indexes = (Index(keys=("date", "ranking")),) diff --git a/old_models/jpep_ftn_macket.py b/old_models/jpep_ftn_macket.py index ad8a88b..ffbd980 100644 --- a/old_models/jpep_ftn_macket.py +++ b/old_models/jpep_ftn_macket.py @@ -1,32 +1,26 @@ from datetime import datetime -from typing import ClassVar, List, Literal, Optional +from typing import Literal, Optional from msgspec import field -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from old_models import OLD_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class OldAmountField(Field, **FIELD_OBJECT_CONFIG): +class OldAmountField(Field, **MODEL_CONFIG): total: int traded: int remaining: int tradable: int -class OldUserField(Field, **FIELD_OBJECT_CONFIG): +class OldUserField(Field, **MODEL_CONFIG): id: int name: str name_md5: Optional[str] = field(name="name_md5") -class OldJPEPFTNMacket(Document, **DOCUMENT_OBJECT_CONFIG): +class OldJPEPFTNMacket(Document, **MODEL_CONFIG): fetch_time: datetime = field(name="fetch_time") order_id: int = field(name="order_id") trade_type: Literal["buy", "sell"] = field(name="trade_type") @@ -42,6 +36,4 @@ class OldJPEPFTNMacket(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = OLD_DB.JPEP_FTN_macket - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["fetch_time"]), - ] + indexes = (Index(keys=("fetch_time",)),) diff --git a/old_models/lottery_data.py b/old_models/lottery_data.py index 4cc1f7b..40e8c90 100644 --- a/old_models/lottery_data.py +++ b/old_models/lottery_data.py @@ -1,25 +1,18 @@ from datetime import datetime -from typing import ClassVar, List from msgspec import field -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from old_models import OLD_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class OldUserField(Field, **FIELD_OBJECT_CONFIG): +class OldUserField(Field, **MODEL_CONFIG): id: int url: str name: str -class OldLotteryData(Document, **DOCUMENT_OBJECT_CONFIG): +class OldLotteryData(Document, **MODEL_CONFIG): _id: int time: datetime reward_name: str = field(name="reward_name") @@ -28,6 +21,4 @@ class OldLotteryData(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = OLD_DB.lottery_data - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["time"]), - ] + indexes = (Index(keys=("time",)),) diff --git a/old_models/lp_collections.py b/old_models/lp_collections.py index 95f47ee..f7cc939 100644 --- a/old_models/lp_collections.py +++ b/old_models/lp_collections.py @@ -1,19 +1,12 @@ from datetime import datetime -from typing import ClassVar, List from msgspec import field -from pymongo import IndexModel +from sshared.mongo import MODEL_CONFIG, Document, Field, Index from old_models import OLD_DB -from utils.document_model import ( - DOCUMENT_OBJECT_CONFIG, - FIELD_OBJECT_CONFIG, - Document, - Field, -) -class OldArticleField(Field, **FIELD_OBJECT_CONFIG): +class OldArticleField(Field, **MODEL_CONFIG): id: int url: str title: str @@ -30,13 +23,13 @@ class OldArticleField(Field, **FIELD_OBJECT_CONFIG): summary: str -class OldAuthorField(Field, **FIELD_OBJECT_CONFIG): +class OldAuthorField(Field, **MODEL_CONFIG): id: int url: str name: str -class OldLPCollections(Document, **DOCUMENT_OBJECT_CONFIG): +class OldLPCollections(Document, **MODEL_CONFIG): fetch_date: datetime = field(name="fetch_date") from_collection: str = field(name="from_collection") @@ -45,6 +38,4 @@ class OldLPCollections(Document, **DOCUMENT_OBJECT_CONFIG): class Meta: # type: ignore collection = OLD_DB.LP_collections - indexes: ClassVar[List[IndexModel]] = [ - IndexModel(["fetch_date"]), - ] + indexes = (Index(keys=("fetch_date",)),) diff --git a/poetry.lock b/poetry.lock index 0b7ca34..f2a4149 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2683,6 +2683,24 @@ postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] pymysql = ["pymysql"] sqlcipher = ["sqlcipher3_binary"] +[[package]] +name = "sshared" +version = "0.1.0" +description = "后端共享组件" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "sshared-0.1.0-py3-none-any.whl", hash = "sha256:927e1cf89d55dfd3285d6d9b2bdd1ee58f5d751b0149a0370bd88fde0c2ae154"}, + {file = "sshared-0.1.0.tar.gz", hash = "sha256:c1313d9db9c31d6b9ee5e200e1c106798e143b4da158c6dfa656c1c0bf6de4da"}, +] + +[package.dependencies] +msgspec = ">=0.18.0,<0.19.0" +typing-extensions = ">=4.10.0,<5.0.0" + +[package.extras] +mongo = ["motor (>=3.3.0,<4.0.0)"] + [[package]] name = "sspeedup" version = "0.25.1" @@ -3031,4 +3049,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "ed1d69514e8ce89ebc241f85ca6bab85898df4b20cdbdc45c0e0bb7183b2713c" +content-hash = "3df33be675ec66d08a18dee2fa43dd12c4780bd1cf2fbe743472c97bb81cdb62" diff --git a/pyproject.toml b/pyproject.toml index c87dab9..046f11b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ python = "^3.8" prefect = "^2.16.0" jkit = "^3.0.0a15" sspeedup = "^0.25.1" +sshared = "^0.1.0" [tool.poetry.group.dev.dependencies] ruff = "^0.3.0" diff --git a/requirements-dev.txt b/requirements-dev.txt index 66f4c9c..a003438 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -95,6 +95,7 @@ six==1.16.0 ; python_version >= "3.8" and python_version < "4.0" sniffio==1.3.1 ; python_version >= "3.8" and python_version < "4.0" sqlalchemy==2.0.28 ; python_version >= "3.8" and python_version < "4.0" sqlalchemy[asyncio]==2.0.28 ; python_version >= "3.8" and python_version < "4.0" +sshared==0.1.0 ; python_version >= "3.8" and python_version < "4.0" sspeedup==0.25.1 ; python_version >= "3.8" and python_version < "4.0" text-unidecode==1.3 ; python_version >= "3.8" and python_version < "4.0" toml==0.10.2 ; python_version >= "3.8" and python_version < "4.0" diff --git a/requirements.txt b/requirements.txt index 0293975..df81b3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -92,6 +92,7 @@ six==1.16.0 ; python_version >= "3.8" and python_version < "4.0" sniffio==1.3.1 ; python_version >= "3.8" and python_version < "4.0" sqlalchemy==2.0.28 ; python_version >= "3.8" and python_version < "4.0" sqlalchemy[asyncio]==2.0.28 ; python_version >= "3.8" and python_version < "4.0" +sshared==0.1.0 ; python_version >= "3.8" and python_version < "4.0" sspeedup==0.25.1 ; python_version >= "3.8" and python_version < "4.0" text-unidecode==1.3 ; python_version >= "3.8" and python_version < "4.0" toml==0.10.2 ; python_version >= "3.8" and python_version < "4.0" diff --git a/utils/document_model.py b/utils/document_model.py deleted file mode 100644 index 4959c20..0000000 --- a/utils/document_model.py +++ /dev/null @@ -1,137 +0,0 @@ -from datetime import datetime -from typing import ( - Any, - AsyncGenerator, - ClassVar, - Dict, - List, - Literal, - Optional, - Sequence, - Tuple, -) - -from bson import ObjectId -from motor.core import AgnosticCollection -from msgspec import Struct, convert, to_builtins -from msgspec.inspect import type_info -from pymongo import IndexModel -from typing_extensions import Self - -FIELD_OBJECT_CONFIG = { - "kw_only": True, - "rename": "camel", -} - -DOCUMENT_OBJECT_CONFIG = { - "kw_only": True, - "rename": "camel", -} - -_BUILDIN_TYPES: Tuple[object, ...] = (ObjectId, datetime) - - -class Field(Struct, **FIELD_OBJECT_CONFIG): - pass - - -class Document(Struct, **DOCUMENT_OBJECT_CONFIG): - class Meta(Struct): - collection: ClassVar[AgnosticCollection] - indexes: ClassVar[List[IndexModel]] - - @classmethod - def _get_field_name(cls, field: object) -> str: - attr_name = field.__qualname__.split(".")[-1] - for field_obj in type_info(cls).fields: # type: ignore - if field_obj.name != attr_name: - continue - - return field_obj.encode_name - - raise ValueError("未找到属性对应的字段名称") - - @classmethod - def get_collection(cls) -> AgnosticCollection: - return cls.Meta.collection - - def validate(self) -> Self: - return convert( - to_builtins(self, builtin_types=_BUILDIN_TYPES), - type=self.__class__, - ) - - @classmethod - def from_dict(cls, data: Dict[str, Any]) -> Self: - if not hasattr(cls, "_id") and "_id" in data: - del data["_id"] - - return convert(data, type=cls) - - def to_dict(self) -> Dict[str, Any]: - return to_builtins( - self, - builtin_types=_BUILDIN_TYPES, - ) - - @classmethod - async def ensure_indexes(cls) -> None: - if not cls.Meta.indexes: - return - - await cls.get_collection().create_indexes(cls.Meta.indexes) - - @classmethod - async def find_one( - cls, - filter: Optional[Dict[str, Any]] = None, # noqa: A002 - /, - *, - sort: Optional[Dict[str, Literal["ASC", "DESC"]]] = None, - ) -> Optional[Self]: - cursor = cls.get_collection().find(filter) - if sort: - cursor = cursor.sort( - {key: 1 if order == "ASC" else -1 for key, order in sort.items()} - ) - cursor = cursor.limit(1) - - try: - return cls.from_dict(await cursor.__anext__()) - except StopAsyncIteration: - return None - - @classmethod - async def find_many( - cls, - filter: Optional[Dict[str, Any]] = None, # noqa: A002 - /, - *, - sort: Optional[Dict[str, Literal["ASC", "DESC"]]] = None, - skip: Optional[int] = None, - limit: Optional[int] = None, - ) -> AsyncGenerator[Self, None]: - cursor = cls.get_collection().find(filter) - if sort: - cursor = cursor.sort( - {key: 1 if order == "ASC" else -1 for key, order in sort.items()} - ) - if skip: - cursor = cursor.skip(skip) - if limit: - cursor = cursor.limit(limit) - - async for item in cursor: - yield cls.from_dict(item) - - @classmethod - async def insert_one(cls, data: Self) -> None: - await cls.get_collection().insert_one(data.to_dict()) - - @classmethod - async def insert_many(cls, data: Sequence[Self]) -> None: - await cls.get_collection().insert_many(x.to_dict() for x in data) - - @classmethod - async def count(cls, filter: Optional[Dict[str, Any]] = None) -> int: # noqa: A002 - return await cls.get_collection().count_documents(filter if filter else {}) diff --git a/utils/migrate_helper.py b/utils/migrate_helper.py index 6abd7ff..567f2cf 100644 --- a/utils/migrate_helper.py +++ b/utils/migrate_helper.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Tuple, Type -from utils.document_model import Document +from sshared.mongo import Document async def get_collection_data_count(document: Type[Document]) -> int: @@ -11,7 +11,7 @@ async def get_collection_data_count(document: Type[Document]) -> int: async def get_collection_data_time_range( document: Type[Document], key: str ) -> Tuple[datetime, datetime]: - start_time = (await document.find_one(sort={key: "ASC"})).__getattribute__(key) - end_time = (await document.find_one(sort={key: "DESC"})).__getattribute__(key) + start_time = (await document.find_one({}, sort={key: "ASC"})).__getattribute__(key) + end_time = (await document.find_one({}, sort={key: "DESC"})).__getattribute__(key) return (start_time, end_time)