diff --git a/jobs/__init__.py b/jobs/__init__.py index d462d26..68fd860 100644 --- a/jobs/__init__.py +++ b/jobs/__init__.py @@ -26,7 +26,6 @@ def import_deployment(path: str) -> DeploymentType: "jobs.jianshu.article_earning_ranking:deployment", "jobs.jianshu.assets_ranking:deployment", "jobs.jianshu.daily_update_ranking:deployment", - "jobs.jianshu.lp_recommend:deployment", "jobs.jpep.ftn_trade:buy_deployment", "jobs.jpep.ftn_trade:sell_deployment", } diff --git a/jobs/jianshu/lp_recommend.py b/jobs/jianshu/lp_recommend.py deleted file mode 100644 index 943126b..0000000 --- a/jobs/jianshu/lp_recommend.py +++ /dev/null @@ -1,99 +0,0 @@ -from datetime import datetime -from typing import Optional - -from jkit.collection import Collection, CollectionArticleInfo -from prefect import flow -from sshared.time import get_today_as_datetime - -from models.jianshu.lp_recommend_article_record import ( - LPRecommendedArticleRecordDocument, -) -from models.jianshu.user import UserDocument -from utils.log import ( - get_flow_run_name, - log_flow_run_start, - log_flow_run_success, - logger, -) -from utils.prefect_helper import ( - generate_deployment_config, - generate_flow_config, -) - -# 理事会点赞汇总专题 -LP_RECOMMENDED_COLLECTION = Collection.from_slug("f61832508891") - - -async def process_item( - item: CollectionArticleInfo, date: datetime -) -> Optional[LPRecommendedArticleRecordDocument]: - flow_run_name = get_flow_run_name() - - if await LPRecommendedArticleRecordDocument.is_record_exist(item.slug): - logger.warn( - "已保存过该文章记录,跳过", - flow_run_name=flow_run_name, - article_slug=item.slug, - ) - return None - - await UserDocument.insert_or_update_one( - slug=item.author_info.slug, - id=item.author_info.id, - name=item.author_info.name, - avatar_url=item.author_info.avatar_url, - ) - - return LPRecommendedArticleRecordDocument( - date=date, - id=item.id, - slug=item.slug, - title=item.title, - published_at=item.published_at, - views_count=item.views_count, - likes_count=item.likes_count, - comments_count=item.comments_count, - tips_count=item.tips_count, - earned_fp_amount=item.earned_fp_amount, - is_paid=item.is_paid, - can_comment=item.can_comment, - description=item.description, - author_slug=item.author_info.slug, - ) - - -@flow( - **generate_flow_config( - name="采集 LP 推荐文章记录", - ) -) -async def main() -> None: - flow_run_name = log_flow_run_start(logger) - - date = get_today_as_datetime() - - data: list[LPRecommendedArticleRecordDocument] = [] - itered_items_count = 0 - async for item in LP_RECOMMENDED_COLLECTION.iter_articles(): - processed_item = await process_item(item, date=date) - if processed_item: - data.append(processed_item) - - itered_items_count += 1 - if itered_items_count == 100: - break - - if data: - await LPRecommendedArticleRecordDocument.insert_many(data) - else: - logger.info("无数据,不执行保存操作", flow_run_name=flow_run_name) - - log_flow_run_success(logger, data_count=len(data)) - - -deployment = main.to_deployment( - **generate_deployment_config( - name="采集 LP 推荐文章记录", - cron="0 1 * * *", - ) -) diff --git a/models/__init__.py b/models/__init__.py index e34c59a..86b84c0 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -4,7 +4,6 @@ from .jianshu.assets_ranking_record import AssetsRankingRecordDocument from .jianshu.daily_update_ranking_record import DailyUpdateRankingRecordDocument from .jianshu.lottery_win_record import LotteryWinRecordDocument -from .jianshu.lp_recommend_article_record import LPRecommendedArticleRecordDocument from .jianshu.user import UserDocument as JianshuUserDocument from .jpep.credit_history import CreditHistoryDocument from .jpep.ftn_trade_order import FTNTradeOrderDocument @@ -15,7 +14,6 @@ AssetsRankingRecordDocument, DailyUpdateRankingRecordDocument, LotteryWinRecordDocument, - LPRecommendedArticleRecordDocument, JianshuUserDocument, CreditHistoryDocument, FTNTradeOrderDocument, diff --git a/models/jianshu/lp_recommend_article_record.py b/models/jianshu/lp_recommend_article_record.py deleted file mode 100644 index 0c84b03..0000000 --- a/models/jianshu/lp_recommend_article_record.py +++ /dev/null @@ -1,42 +0,0 @@ -from datetime import datetime - -from jkit.msgspec_constraints import ( - ArticleSlug, - NonEmptyStr, - NonNegativeFloat, - NonNegativeInt, - PositiveInt, - UserSlug, -) -from msgspec import field -from sshared.mongo import Document, Index - -from utils.mongo import JIANSHU_DB - - -class LPRecommendedArticleRecordDocument(Document, frozen=True): - date: datetime - id: PositiveInt - slug: ArticleSlug - title: NonEmptyStr - published_at: datetime - - views_count: NonNegativeInt - likes_count: NonNegativeInt - comments_count: NonNegativeInt - tips_count: NonNegativeFloat - earned_fp_amount: NonNegativeFloat = field(name="earnedFPAmount") - - is_paid: bool - can_comment: bool - description: str - - author_slug: UserSlug - - class Meta: # type: ignore - collection = JIANSHU_DB.lp_recommended_article_records - indexes = (Index(keys=("date", "slug"), unique=True),) - - @classmethod - async def is_record_exist(cls, slug: str) -> bool: - return await cls.find_one({"slug": slug}) is not None