Skip to content

Commit

Permalink
pref: 使用线程池优化数据库操作性能
Browse files Browse the repository at this point in the history
  • Loading branch information
@FHU-yezi
FHU-yezi committed Nov 11, 2024
1 parent 7b7fa5c commit c6ee66f
Show file tree
Hide file tree
Showing 11 changed files with 378 additions and 360 deletions.
4 changes: 4 additions & 0 deletions models/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,13 @@
from models.jpep.ftn_macket_record import FTNMacketRecord
from models.jpep.ftn_order import FTNOrder
from models.jpep.user import User as JPEPUser
from utils.db import jianshu_pool, jpep_pool


async def init_db() -> None:
await jianshu_pool.prepare()
await jpep_pool.prepare()

await ArticleEarningRankingRecord.init()
await DailyUpdateRankingRecord.init()
await JianshuUser.init()
Expand Down
68 changes: 34 additions & 34 deletions models/jianshu/article_earning_ranking_record.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from sshared.postgres import Table
from sshared.strict_struct import NonEmptyStr, PositiveFloat, PositiveInt

from utils.postgres import get_jianshu_conn
from utils.db import jianshu_pool


class ArticleEarningRankingRecord(Table, frozen=True):
Expand All @@ -19,42 +19,42 @@ class ArticleEarningRankingRecord(Table, frozen=True):

@classmethod
async def _create_table(cls) -> None:
conn = await get_jianshu_conn()
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS article_earning_ranking_records (
date DATE NOT NULL,
ranking SMALLINT NOT NULL,
slug VARCHAR(12),
title TEXT,
author_slug VARCHAR(12),
author_earning NUMERIC NOT NULL,
voter_earning NUMERIC NOT NULL,
CONSTRAINT pk_article_earning_ranking_records_date_ranking PRIMARY KEY (date, ranking)
);
""" # noqa: E501
)
async with jianshu_pool.get_conn() as conn:
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS article_earning_ranking_records (
date DATE NOT NULL,
ranking SMALLINT NOT NULL,
slug VARCHAR(12),
title TEXT,
author_slug VARCHAR(12),
author_earning NUMERIC NOT NULL,
voter_earning NUMERIC NOT NULL,
CONSTRAINT pk_article_earning_ranking_records_date_ranking PRIMARY KEY (date, ranking)
);
""" # noqa: E501
)

@classmethod
async def insert_many(cls, data: list["ArticleEarningRankingRecord"]) -> None:
for item in data:
item.validate()

conn = await get_jianshu_conn()
await conn.cursor().executemany(
"INSERT INTO article_earning_ranking_records (date, ranking, "
"slug, title, author_slug, author_earning, voter_earning) "
"VALUES (%s, %s, %s, %s, %s, %s, %s);",
[
(
item.date,
item.ranking,
item.slug,
item.title,
item.author_slug,
item.author_earning,
item.voter_earning,
)
for item in data
],
)
async with jianshu_pool.get_conn() as conn:
await conn.cursor().executemany(
"INSERT INTO article_earning_ranking_records (date, ranking, "
"slug, title, author_slug, author_earning, voter_earning) "
"VALUES (%s, %s, %s, %s, %s, %s, %s);",
[
(
item.date,
item.ranking,
item.slug,
item.title,
item.author_slug,
item.author_earning,
item.voter_earning,
)
for item in data
],
)
54 changes: 27 additions & 27 deletions models/jianshu/daily_update_ranking_record.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from sshared.postgres import Table
from sshared.strict_struct import NonEmptyStr, PositiveInt

from utils.postgres import get_jianshu_conn
from utils.db import jianshu_pool


class DailyUpdateRankingRecord(Table, frozen=True):
Expand All @@ -14,35 +14,35 @@ class DailyUpdateRankingRecord(Table, frozen=True):

@classmethod
async def _create_table(cls) -> None:
conn = await get_jianshu_conn()
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS daily_update_ranking_records (
date DATE NOT NULL,
ranking SMALLINT NOT NULL,
slug VARCHAR(12),
days SMALLINT,
CONSTRAINT pk_daily_update_ranking_records_date_slug PRIMARY KEY (date, slug)
);
""" # noqa: E501
)
async with jianshu_pool.get_conn() as conn:
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS daily_update_ranking_records (
date DATE NOT NULL,
ranking SMALLINT NOT NULL,
slug VARCHAR(12),
days SMALLINT,
CONSTRAINT pk_daily_update_ranking_records_date_slug PRIMARY KEY (date, slug)
);
""" # noqa: E501
)

@classmethod
async def insert_many(cls, data: list["DailyUpdateRankingRecord"]) -> None:
for item in data:
item.validate()

conn = await get_jianshu_conn()
await conn.cursor().executemany(
"INSERT INTO daily_update_ranking_records (date, ranking, "
"slug, days) VALUES (%s, %s, %s, %s);",
[
(
item.date,
item.ranking,
item.slug,
item.days,
)
for item in data
],
)
async with jianshu_pool.get_conn() as conn:
await conn.cursor().executemany(
"INSERT INTO daily_update_ranking_records (date, ranking, "
"slug, days) VALUES (%s, %s, %s, %s);",
[
(
item.date,
item.ranking,
item.slug,
item.days,
)
for item in data
],
)
174 changes: 88 additions & 86 deletions models/jianshu/user.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from sshared.postgres import Table, create_enum
from sshared.strict_struct import NonEmptyStr, PositiveInt

from utils.postgres import get_jianshu_conn
from utils.db import jianshu_pool


class StatusEnum(Enum):
Expand All @@ -24,54 +24,56 @@ class User(Table, frozen=True):

@classmethod
async def _create_enum(cls) -> None:
conn = await get_jianshu_conn()
await create_enum(conn=conn, name="enum_users_status", enum_class=StatusEnum)
async with jianshu_pool.get_conn() as conn:
await create_enum(
conn=conn, name="enum_users_status", enum_class=StatusEnum
)

@classmethod
async def _create_table(cls) -> None:
conn = await get_jianshu_conn()
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS users (
slug VARCHAR(12) CONSTRAINT pk_users_slug PRIMARY KEY,
status enum_users_status NOT NULL,
update_time TIMESTAMP NOT NULL,
id INTEGER,
name VARCHAR(15),
history_names VARCHAR(15)[] NOT NULL,
avatar_url TEXT
);
"""
)
async with jianshu_pool.get_conn() as conn:
await conn.execute(
"""
CREATE TABLE IF NOT EXISTS users (
slug VARCHAR(12) CONSTRAINT pk_users_slug PRIMARY KEY,
status enum_users_status NOT NULL,
update_time TIMESTAMP NOT NULL,
id INTEGER,
name VARCHAR(15),
history_names VARCHAR(15)[] NOT NULL,
avatar_url TEXT
);
"""
)

async def create(self) -> None:
self.validate()

conn = await get_jianshu_conn()
await conn.execute(
"INSERT INTO users (slug, status, update_time, id, name, "
"history_names, avatar_url) VALUES (%s, %s, %s, %s, %s, %s, %s);",
(
self.slug,
self.status,
self.update_time,
self.id,
self.name,
self.history_names,
self.avatar_url,
),
)
async with jianshu_pool.get_conn() as conn:
await conn.execute(
"INSERT INTO users (slug, status, update_time, id, name, "
"history_names, avatar_url) VALUES (%s, %s, %s, %s, %s, %s, %s);",
(
self.slug,
self.status,
self.update_time,
self.id,
self.name,
self.history_names,
self.avatar_url,
),
)

@classmethod
async def get_by_slug(cls, slug: str) -> Optional["User"]:
conn = await get_jianshu_conn()
cursor = await conn.execute(
"SELECT status, update_time, id, name, history_names, "
"avatar_url FROM users WHERE slug = %s;",
(slug,),
)
async with jianshu_pool.get_conn() as conn:
cursor = await conn.execute(
"SELECT status, update_time, id, name, history_names, "
"avatar_url FROM users WHERE slug = %s;",
(slug,),
)

data = await cursor.fetchone()
data = await cursor.fetchone()
if not data:
return None

Expand Down Expand Up @@ -112,54 +114,54 @@ async def upsert(
return

# 在一个事务中一次性完成全部字段的更新
conn = await get_jianshu_conn()
async with conn.transaction():
# 更新更新时间
await conn.execute(
"UPDATE users SET update_time = %s WHERE slug = %s",
(datetime.now(), slug),
)

# ID 无法被修改，如果异常则抛出错误
if user.id and id and user.id != id:
raise ValueError(f"用户 ID 不一致：{user.id} != {id}")

# 如果没有存储 ID，进行添加
if not user.id and id:
async with jianshu_pool.get_conn() as conn: # noqa: SIM117
async with conn.transaction():
# 更新更新时间
await conn.execute(
"UPDATE users SET id = %s WHERE slug = %s",
(id, slug),
"UPDATE users SET update_time = %s WHERE slug = %s",
(datetime.now(), slug),
)

# 如果没有存储昵称，进行添加
if not user.name and name:
await conn.execute(
"UPDATE users SET name = %s WHERE slug = %s",
(name, slug),
)

# 更新昵称
if user.name and name and user.name != name:
await conn.execute(
"UPDATE users SET name = %s WHERE slug = %s",
(name, slug),
)
await conn.execute(
"UPDATE users SET history_names = array_append(history_names, %s) "
"WHERE slug = %s;",
(user.name, slug),
)

# 如果没有存储头像链接，进行添加
if not user.avatar_url and avatar_url:
await conn.execute(
"UPDATE users SET avatar_url = %s WHERE slug = %s",
(avatar_url, slug),
)

# 更新头像链接
if user.avatar_url and avatar_url and user.avatar_url != avatar_url:
await conn.execute(
"UPDATE users SET avatar_url = %s WHERE slug = %s",
(avatar_url, slug),
)
# ID 无法被修改，如果异常则抛出错误
if user.id and id and user.id != id:
raise ValueError(f"用户 ID 不一致：{user.id} != {id}")

# 如果没有存储 ID，进行添加
if not user.id and id:
await conn.execute(
"UPDATE users SET id = %s WHERE slug = %s",
(id, slug),
)

# 如果没有存储昵称，进行添加
if not user.name and name:
await conn.execute(
"UPDATE users SET name = %s WHERE slug = %s",
(name, slug),
)

# 更新昵称
if user.name and name and user.name != name:
await conn.execute(
"UPDATE users SET name = %s WHERE slug = %s",
(name, slug),
)
await conn.execute(
"UPDATE users SET history_names = array_append(history_names, "
"%s) WHERE slug = %s;",
(user.name, slug),
)

# 如果没有存储头像链接，进行添加
if not user.avatar_url and avatar_url:
await conn.execute(
"UPDATE users SET avatar_url = %s WHERE slug = %s",
(avatar_url, slug),
)

# 更新头像链接
if user.avatar_url and avatar_url and user.avatar_url != avatar_url:
await conn.execute(
"UPDATE users SET avatar_url = %s WHERE slug = %s",
(avatar_url, slug),
)
Loading

0 comments on commit c6ee66f

Please sign in to comment.