diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..1957188 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,18 @@ +{ + "image": "mcr.microsoft.com/devcontainers/python:3.12", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "GitHub.copilot", + "ms-python.mypy-type-checker", + "ms-python.vscode-pylance", + "ms-python.debugpy", + "charliermarsh.ruff", + "vscode-icons-team.vscode-icons" + ] + } + }, + "forwardPorts": [8000], + "postCreateCommand": "pipx install pdm; pdm install;" +} diff --git a/.env b/.env new file mode 100644 index 0000000..bff769d --- /dev/null +++ b/.env @@ -0,0 +1,10 @@ +SERVER_URL='https://example.com' + +PGHOST='*.neon.tech' +PGDATABASE='db_name' +PGUSER='user_name' +PGPASSWORD='' + +OPENAI_API_KEY='sk-xxx' + +CO_API_KEY='xxxxx' \ No newline at end of file diff --git a/.env.template b/.env.template deleted file mode 100644 index 2857172..0000000 --- a/.env.template +++ /dev/null @@ -1,5 +0,0 @@ -PGHOST= -PGDATABASE= -PGUSER= -PGPASSWORD= -OPENAI_API_KEY= \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 7537a92..21cb559 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,8 +4,20 @@ "files.exclude": { "**/__pycache__": true, "**/.mypy_cache": true, - ".pdm-python": true, + "**/.pdm-python": true, "**/.pdm-build": true, - ".venv": true - } + "**/.venv": true, + "**/.idea": true, + "**/.ruff_cache": true, + "**/.pytest_cache": true, + }, + "python.analysis.inlayHints.functionReturnTypes": true, + "python.analysis.inlayHints.variableTypes": true, + "python.analysis.inlayHints.pytestParameters": true, + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": [ + "packages" + ], + "terminal.integrated.shellIntegration.history": 100000, } \ No newline at end of file diff --git a/README.md b/README.md index 2c5dd18..7025f74 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,66 @@ -# loci - -> The method of loci is a strategy for memory enhancement, -> which uses visualizations of familiar spatial environments in order to enhance the recall of information. -> The method of loci is also known as the memory journey, -> memory palace, journey method, memory spaces, or mind palace technique. -> -> -- wikipedia - +# koma + ## Overview Notice: This project is still in development. -This project(`loci`) is designed to help users search for information on macOS. +This project(`koma`) is designed to help users search for information on macOS. -Specifically thanks to the [apple_cloud_notes_parser](https://github.com/threeplanetssoftware/apple_cloud_notes_parser). +Specifically thanks to the [apple_cloud_notes_parser](https://github.com/threeplanetssoftware/apple_cloud_notes_parser) and [neon](https://neon.tech). ## Feature -in progress: +- [x] [core] List all notes from Notes.app; +- [x] [core] Convert all notes into markdown; +- [ ] [core] Get a specific note; +- [ ] [core] Create a note using AppleScript; +- [x] [api ] Incremental index all notes and its paragraphs; +- [x] [api ] Semantic query in both json and plain text; +- [ ] [api ] Summrize similar content; +- [ ] [api ] Automatic sync Apple notes. + +## Usage + +### As a data connector in RAG platform. + +By using OpenAPI this is easy to intergate into any rag platform, such as [Dify](https://dify.ai). + +In Dify, goto [Tools](https://cloud.dify.ai/tools?category=api) and create one using `/api/openapi.json`. + +### Raycast + +TODO. + +## Install + +### Step 0: Prepare + +1. You have [pdm](https://pdm-project.org/en/stable/) and [uvicorn](https://www.uvicorn.org) installed. +2. Your Terminal have [Full Disk Access](https://www.perplexity.ai/search/How-to-enable-mOAW4vpVRlmeMvtg6EjnNw) permission. +3. Your Apple Notes.app folder is `~/Library/Group Containers/group.com.apple.notes` +4. Configure the `.env` + +### Step 1: download this project + +``` +> git clone https://github.com/AFutureD/koma.git +``` + +### Step 2: Install dependencies. + +``` +> cd koma +> pdm sync +``` -- [x] [core] get notes from notes.app -- [x] [core] render notes into markdown -- [x] [core] incremental save notes into database(current mongodb only) -- [x] [cli ] list notes -- [ ] [cli ] get the content of a note from database -- [ ] [cli ] search notes +### Step 3: Run server +``` +> pdm run django_manage migrate rag +> uvicorn agent.asgi:application --host 0.0.0.0 --env-file ./.env +``` -TODO: -- [ ] [core] using pyiCloud. diff --git a/config_template.toml b/config_template.toml deleted file mode 100644 index 57839c7..0000000 --- a/config_template.toml +++ /dev/null @@ -1,2 +0,0 @@ -[database] -mongodb.uri = "mongodb+srv://username:password@domain.com" \ No newline at end of file diff --git a/packages/pkg-cli/pyproject.toml b/packages/pkg-cli/pyproject.toml index e8df15e..de937f6 100644 --- a/packages/pkg-cli/pyproject.toml +++ b/packages/pkg-cli/pyproject.toml @@ -8,7 +8,7 @@ authors = [ dependencies = [ "click>=8.1.7", "tomlkit>=0.12.4", - "loci" + "koma" ] requires-python = ">=3.12" readme = "README.md" diff --git a/packages/pkg-core/loci/__init__.py b/packages/pkg-core/koma/__init__.py similarity index 100% rename from packages/pkg-core/loci/__init__.py rename to packages/pkg-core/koma/__init__.py diff --git a/packages/pkg-core/koma/core/__init__.py b/packages/pkg-core/koma/core/__init__.py new file mode 100644 index 0000000..af1f346 --- /dev/null +++ b/packages/pkg-core/koma/core/__init__.py @@ -0,0 +1,6 @@ +from .render import RenderAble, TextRenderer +from .model import Model + +__all__ = [ + 'Model', 'TextRenderer', 'RenderAble' +] \ No newline at end of file diff --git a/packages/pkg-core/koma/core/model.py b/packages/pkg-core/koma/core/model.py new file mode 100644 index 0000000..0706bb9 --- /dev/null +++ b/packages/pkg-core/koma/core/model.py @@ -0,0 +1,6 @@ +from typing import Any +from pydantic import BaseModel, Field + +class Model(BaseModel): + metadata: dict[str, Any] = Field(default=dict(), exclude=True) + diff --git a/packages/pkg-core/loci/core/render.py b/packages/pkg-core/koma/core/render.py similarity index 53% rename from packages/pkg-core/loci/core/render.py rename to packages/pkg-core/koma/core/render.py index e07744f..eb069e0 100644 --- a/packages/pkg-core/loci/core/render.py +++ b/packages/pkg-core/koma/core/render.py @@ -1,24 +1,27 @@ from __future__ import annotations + from abc import ABC, abstractmethod +from pydantic import Field -from pydantic import BaseModel +from .model import Model -class BaseRenderer(ABC): +class TextRenderer(ABC): + META_KEY = "TEXT_RENDER_RESULT" @abstractmethod def render(self, renderable: RenderAble) -> str: - pass + ... -class RenderAble(BaseModel): - rendered: bool = False +class RenderAble(Model): - represent: str | None = None + rendered: bool = Field(default=False, exclude=True) + rendered_result: str | None = Field(default=None, exclude=True) - def render(self, renderer: BaseRenderer): + def render(self, renderer: TextRenderer): if self.rendered: - return self.represent + return for prop in self.__dict__.values(): if isinstance(prop, RenderAble): @@ -33,8 +36,9 @@ def render(self, renderer: BaseRenderer): item.render(renderer) self.rendered = True - if isinstance(self, RenderAble): - represent = renderer.render(self) - self.represent = represent - else: - raise RuntimeError(f"Unknown type {type(self)}") \ No newline at end of file + assert isinstance(self, RenderAble), f"{self.__class__.__qualname__} must be RenderAble" + + render_result = renderer.render(self) + + self.metadata[renderer.META_KEY] = render_result + self.rendered_result = render_result \ No newline at end of file diff --git a/packages/pkg-core/loci/domain/__init__.py b/packages/pkg-core/koma/domain/__init__.py similarity index 85% rename from packages/pkg-core/loci/domain/__init__.py rename to packages/pkg-core/koma/domain/__init__.py index 3a4ee39..ba4ee66 100644 --- a/packages/pkg-core/loci/domain/__init__.py +++ b/packages/pkg-core/koma/domain/__init__.py @@ -1,6 +1,6 @@ from .models.note import Note, NoteContent, NoteContentLine, NoteContentParagraph from .models.attachment import NoteAttachment, NoteAttachmentLink, NoteAttachmentMedia, NoteAttachmentTag, NoteAttachmentDraw, NoteAttachmentGallery, NoteAttachmentTable, NoteAttachmentTableCell -from .models.text import ParagraphStyleType, FontStyle, CheckInfo, ParagraphStyle, TextAttribute, AttributeText +from .models.style import ParagraphStyleType, FontStyle, CheckInfo, ParagraphStyle, TextAttribute, AttributeText __all__ = [ "ParagraphStyleType", "FontStyle", "CheckInfo", "ParagraphStyle", "TextAttribute", "AttributeText", diff --git a/packages/pkg-core/loci/domain/models/__init__.py b/packages/pkg-core/koma/domain/models/__init__.py similarity index 100% rename from packages/pkg-core/loci/domain/models/__init__.py rename to packages/pkg-core/koma/domain/models/__init__.py diff --git a/packages/pkg-core/loci/domain/models/attachment.py b/packages/pkg-core/koma/domain/models/attachment.py similarity index 68% rename from packages/pkg-core/loci/domain/models/attachment.py rename to packages/pkg-core/koma/domain/models/attachment.py index 0eac4fd..0113627 100644 --- a/packages/pkg-core/loci/domain/models/attachment.py +++ b/packages/pkg-core/koma/domain/models/attachment.py @@ -1,15 +1,15 @@ from __future__ import annotations from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Protocol from pydantic import BaseModel, ConfigDict from pydantic._internal._model_construction import ModelMetaclass -from ...core import RenderAble +from ...core import RenderAble, Model -class NoteAttachmentTableCell(BaseModel): +class NoteAttachmentTableCell(Model): column: int row: int text: str @@ -23,19 +23,20 @@ def __repr__(self): class NoteAttachmentFactory(ModelMetaclass): - def __call__(cls, type_uti: str, *args, **kwargs): + def __call__(cls, *args, **kwargs): + type_uti = kwargs.get("type_uti") if type_uti == "public.url": - return NoteAttachmentLink(type_uti = type_uti, **kwargs) + return NoteAttachmentLink(**kwargs) elif type_uti in ["com.apple.paper", "com.apple.drawing.2", "com.apple.drawing"]: - return NoteAttachmentDraw(type_uti = type_uti, **kwargs) + return NoteAttachmentDraw(**kwargs) elif type_uti in ["com.apple.notes.table"]: - return NoteAttachmentTable(type_uti = type_uti, **kwargs) + return NoteAttachmentTable(**kwargs) elif type_uti in ["com.apple.notes.inlinetextattachment.hashtag"]: - return NoteAttachmentTag(type_uti = type_uti, **kwargs) + return NoteAttachmentTag(**kwargs) elif type_uti == "com.apple.notes.gallery": - return NoteAttachmentGallery(type_uti = type_uti, **kwargs) + return NoteAttachmentGallery(**kwargs) else: - return NoteAttachmentMedia(type_uti = type_uti, **kwargs) + return NoteAttachmentMedia(**kwargs) class NoteAttachmentMetaClass(NoteAttachmentFactory): @@ -44,7 +45,7 @@ def __call__(cls, *args, **kwargs): return type.__call__(cls, *args, **kwargs) -class NoteAttachment(RenderAble, BaseModel, metaclass=NoteAttachmentFactory): +class NoteAttachment(RenderAble, Model, metaclass=NoteAttachmentFactory): type_uti: str z_pk: int @@ -88,16 +89,14 @@ class NoteAttachmentMedia(NoteAttachment, metaclass=NoteAttachmentMetaClass): file_uuid: str file_name: str - def get_data_path(self) -> None | str: - if self.media_root_path is None: - return "" - - root = Path(self.media_root_path) + def get_data_path(self) -> str: + if self.generation is None: - data_path = root / "Media" / self.file_uuid / self.file_name + data_path = f"Media/{self.file_uuid}/{self.file_name}" else: - data_path = root / "Media" / self.file_uuid / self.generation / self.file_name - return str(data_path.absolute()) + data_path = f"Media/{self.file_uuid}/{self.generation}/{self.file_name}" + + return f"{self.media_root_path}/{data_path}" class NoteAttachmentDraw(NoteAttachment, metaclass=NoteAttachmentMetaClass): @@ -105,13 +104,13 @@ class NoteAttachmentDraw(NoteAttachment, metaclass=NoteAttachmentMetaClass): media_root_path: str def get_data_path(self) -> str: - root = Path(self.media_root_path) if self.generation is None: - data_path = root / "FallbackImages" / "{}.jpg".format(self.identifier) + data_path = f"FallbackImages/{self.identifier}.jpg" else: - data_path = root / "FallbackImages" / self.identifier / self.generation / "FallbackImage.png" - return str(data_path.absolute()) + data_path = f"FallbackImages/{self.identifier}/{self.generation}/FallbackImage.png" + + return f"{self.media_root_path}/{data_path}" class NoteAttachmentTable(NoteAttachment, metaclass=NoteAttachmentMetaClass): @@ -123,5 +122,6 @@ class NoteAttachmentTable(NoteAttachment, metaclass=NoteAttachmentMetaClass): class NoteAttachmentTag(NoteAttachment, metaclass=NoteAttachmentMetaClass): pass + class NoteAttachmentGallery(NoteAttachment, metaclass=NoteAttachmentMetaClass): pass diff --git a/packages/pkg-core/loci/domain/models/note.py b/packages/pkg-core/koma/domain/models/note.py similarity index 60% rename from packages/pkg-core/loci/domain/models/note.py rename to packages/pkg-core/koma/domain/models/note.py index d7855b8..0edf87e 100644 --- a/packages/pkg-core/loci/domain/models/note.py +++ b/packages/pkg-core/koma/domain/models/note.py @@ -2,33 +2,21 @@ import itertools from datetime import datetime -from typing import Sequence, Annotated +from typing import Sequence -from pydantic import BaseModel, SkipValidation, ConfigDict +from pydantic import BaseModel -from .text import ParagraphStyle, AttributeText -from ...core import RenderAble +from .style import ParagraphStyle, AttributeText +from ...core import RenderAble, Model -class NoteContentLine(RenderAble, BaseModel): +class NoteContentLine(RenderAble, Model): idx: int elements: Sequence[AttributeText] plain_text: str attribute: None | ParagraphStyle - # def __init__(self, idx: int, elements: Sequence[AttributeText]): - # rebuilt_elements = self._rebuild_elements(elements) - # plain_text = "".join([element.text for element in rebuilt_elements]) - # attribute = self._build_attribute(rebuilt_elements) - # - # super().__init__(idx = idx, elements = elements, plain_text = plain_text, attribute = attribute) - # - # self.idx = idx - # self.elements = rebuilt_elements - # self.plain_text = plain_text - # self.attribute = attribute - @classmethod def of(cls, idx: int, elements: Sequence[AttributeText]) -> NoteContentLine: rebuilt_elements = cls._rebuild_elements(elements) @@ -70,38 +58,44 @@ def _build_attribute(elements: Sequence[AttributeText]) -> ParagraphStyle | None def __str__(self): return self.plain_text.__repr__() - def is_same_paragraph(self, other: NoteContentLine | None): - if other is None: + def __repr__(self): + return f"{self.__class__.__name__}({self.model_dump(exclude_none=True)})" + + def is_same_paragraph(self, previous: NoteContentLine | None): + if previous is None: return False - if self.attribute is None and other.attribute is None: + if self.attribute is None and previous.attribute is None: return True - elif self.attribute is not None and other.attribute is None: + elif self.attribute is not None and previous.attribute is None: return False - elif self.attribute is None and other.attribute is not None: + elif self.attribute is None and previous.attribute is not None: return False else: - assert other.attribute is not None + assert previous.attribute is not None assert self.attribute is not None - return self.attribute.is_same_paragraph(other.attribute) + return self.attribute.is_same_block(previous.attribute) def is_paragraph_breaker(self): return self.plain_text == "\n" -class NoteContentParagraph(RenderAble, BaseModel): +class NoteContentParagraph(RenderAble, Model): lines: Sequence[NoteContentLine] attribute: None | ParagraphStyle + def __repr__(self) -> str: + return f"{self.__class__.__name__}({self.model_dump(exclude_none=True)})" + -class NoteContent(RenderAble, BaseModel): +class NoteContent(RenderAble, Model): plan_text: str attributed_text: Sequence[AttributeText] paragraph_list: Sequence[NoteContentParagraph] -class Note(RenderAble, BaseModel): +class Note(RenderAble, Model): z_pk: int uuid: str navigation_link: str @@ -118,8 +112,5 @@ def __str__(self): return self.__repr__() def __repr__(self): - return (f"Memory(z_pk: {self.z_pk}, uuid: {self.uuid}, modified_at: {self.modified_at}, " - f"locked: {self.locked}, pinned: {self.pinned}, title: {self.title}, " - f"folder_name: {self.folder_name}, account_name: {self.account_pk}, " - f"url_scheme: `{self.navigation_link}`") + return f"{self.__class__.__name__}({self.model_dump(exclude=["content", "represent"], exclude_none=True)})" diff --git a/packages/pkg-core/loci/domain/models/text.py b/packages/pkg-core/koma/domain/models/style.py similarity index 66% rename from packages/pkg-core/loci/domain/models/text.py rename to packages/pkg-core/koma/domain/models/style.py index 62453e5..35ae65e 100644 --- a/packages/pkg-core/loci/domain/models/text.py +++ b/packages/pkg-core/koma/domain/models/style.py @@ -6,11 +6,10 @@ from pydantic import BaseModel from .attachment import NoteAttachment -from ...core import RenderAble, BaseRenderer +from ...core import RenderAble, TextRenderer, Model class ParagraphStyleType(IntEnum): - Plain = -1 Title = 0 Heading = 1 Subheading = 2 @@ -22,7 +21,11 @@ class ParagraphStyleType(IntEnum): TodoList = 103 def group_identifier(self): - return self.value if self.value < 100 else 100 + return self.value if not self.should_newline else 100 + + @property + def should_newline(self): + return self.value >= 100 class FontStyle(IntEnum): @@ -32,7 +35,7 @@ class FontStyle(IntEnum): Bold_Italic = 3 -class CheckInfo(BaseModel): +class CheckInfo(Model): done: bool uuid: str @@ -46,28 +49,30 @@ def __repr__(self): return "CheckInfo(done: {}, uuid: {})".format(self.done, self.uuid) -class ParagraphStyle(BaseModel): +class ParagraphStyle(Model): style_type: None | ParagraphStyleType = None indent_level: int = 0 check_info: None | CheckInfo = None quote: bool = False - def is_same_paragraph(self, other: ParagraphStyle): + def is_same_block(self, previous: ParagraphStyle): # two lines are to-do list - if self.check_info is not None and other.check_info is not None: + if self.check_info is not None and previous.check_info is not None: return True # two lines are quote - if self.quote and self.quote: - return True + if self.quote or previous.quote: + return self.quote and previous.quote - if self.style_type is None and other.style_type is None: + if self.style_type is None or previous.style_type is None: return True - elif self.style_type is not None and other.style_type is None: - return self.style_type.group_identifier() >= ParagraphStyleType.DotList.group_identifier() - elif self.style_type is not None and other.style_type is not None: - return self.style_type.group_identifier() <= other.style_type.group_identifier() + # elif self.style_type is not None and previous.style_type is None: + # return True + # elif self.style_type is None and previous.style_type is not None: + # return True + elif self.style_type is not None and previous.style_type is not None: + return self.style_type.group_identifier() >= previous.style_type.group_identifier() else: return False @@ -78,15 +83,16 @@ def __repr__(self): return f"DocParagraphStyle(style_type: {self.style_type}, indent_level: {self.indent_level}, check_info: {self.check_info}, quote: {self.quote})" -class TextAttribute(BaseModel): - paragraph_style: None | ParagraphStyle +class TextAttribute(Model): + paragraph_style: None | ParagraphStyle = None - font_style: None | FontStyle - font_name: None | str # useless for now - underlined: None | bool - strike_through: None | bool - link: None | str - attachment: None | NoteAttachment + font_style: None | FontStyle = None + font_name: None | str = None # useless for now + underlined: None | bool = None + strike_through: None | bool = None + link: None | str = None + attachment_identifier: None | str = None + attachment: None | NoteAttachment = None def __hash__(self): return hash( @@ -101,7 +107,7 @@ def __str__(self): return self.__repr__() def __repr__(self): - return f"NoteAttribute(paragraph: {self.paragraph_style}, font_style: {self.font_style}, underlined: {self.underlined}, strike: {self.strike_through}, link: {self.link.__repr__()}, attachment: {self.attachment})" + return f"NoteAttribute({self.model_dump(exclude_none=True)})" def inline_identifier(self): return self.__hash__() @@ -121,21 +127,13 @@ def __eq__(self, other): ) -class AttributeText(RenderAble, BaseModel): +class AttributeText(RenderAble, Model): start_index: int length: int text: str - # __length_utf_16: int - # __text_utf_16: bytes attribute: TextAttribute - # @classmethod - # def of(cls, start_index: int, length: int, text: str, attribute: TextAttribute): - # text_utf_16 = text.encode("utf-16le") - # - # return AttributeText(start_index= start_index, length = length, text = text, attribute = attribute) - def __str__(self): return self.__repr__() @@ -144,7 +142,7 @@ def __repr__(self): self.text.__repr__(), self.start_index, self.length, self.attribute ) - def render(self, renderer: BaseRenderer): + def render(self, renderer: TextRenderer): if self.attribute and self.attribute.attachment: self.attribute.attachment.render(renderer) diff --git a/packages/pkg-core/loci/infra/__init__.py b/packages/pkg-core/koma/infra/__init__.py similarity index 100% rename from packages/pkg-core/loci/infra/__init__.py rename to packages/pkg-core/koma/infra/__init__.py diff --git a/packages/pkg-core/loci/infra/fetchers/__init__.py b/packages/pkg-core/koma/infra/fetchers/__init__.py similarity index 100% rename from packages/pkg-core/loci/infra/fetchers/__init__.py rename to packages/pkg-core/koma/infra/fetchers/__init__.py diff --git a/packages/pkg-core/loci/infra/fetchers/apple.py b/packages/pkg-core/koma/infra/fetchers/apple.py similarity index 65% rename from packages/pkg-core/loci/infra/fetchers/apple.py rename to packages/pkg-core/koma/infra/fetchers/apple.py index 0cfdbde..09551ac 100644 --- a/packages/pkg-core/loci/infra/fetchers/apple.py +++ b/packages/pkg-core/koma/infra/fetchers/apple.py @@ -1,16 +1,31 @@ import itertools +import logging import sqlite3 import zlib from datetime import datetime, timezone -from typing import MutableSequence, override, Sequence, List, Dict from pathlib import Path +from typing import Dict, List, MutableSequence, Sequence, override -from ...domain import Note, NoteContent, NoteContentParagraph, NoteContentLine, NoteAttachment -from ...domain import TextAttribute, ParagraphStyle, FontStyle, ParagraphStyleType, CheckInfo, AttributeText -from ...infra.helper import AppleNotesTableConstructor -from ...protobuf import NoteStoreProto, Checklist, AttributeRun, MergableDataProto, ParagraphStyle_pb2 -from .base import BaseFetcher + +from ...domain import ( + AttributeText, + Note, + NoteAttachment, + NoteContent, + NoteContentLine +) +from ...infra.helper import AppleNotesTableConstructor, build_note_attribute, build_paragraph_list, build_content_lines from ...infra.renderers import Renderer +from ...protobuf import ( + AttributeRun, + MergableDataProto, + NoteStoreProto, +) +from .base import BaseFetcher + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + ALL_NOTES_QUERY = """ SELECT @@ -118,7 +133,7 @@ class AppleNotesFetcher(BaseFetcher): connection: sqlite3.Connection attachments: List[NoteAttachment] = [] - attachment_by_identify_map: Dict[str, NoteAttachment] = {} + attachment_map_by_identifier: Dict[str, NoteAttachment] = {} accounts: List[AppleNotesAccount] = [] account_by_pk_map: Dict[int, AppleNotesAccount] = {} @@ -141,10 +156,9 @@ def start_fetch(self) -> List[Note]: @override def finish(self): - # for note in self.notes: - # # print(memory) - # print(note.represent) - pass + for note in self.notes: + logger.info(f"{note.__repr__()}") + def list_account_list(self): account_rows = [] @@ -207,7 +221,7 @@ def list_all_attachments(self): attachments.append(attachment) self.attachments = attachments - self.attachment_by_identify_map = {attachment.identifier: attachment for attachment in attachments} + self.attachment_map_by_identifier = {attachment.identifier: attachment for attachment in attachments} # Build attachment hierarchy # for now, we only care about `gallery` type @@ -215,10 +229,6 @@ def list_all_attachments(self): # └─ image # └─ media <- build this by sql join def build_attachment_hierarchy(self): - # grouped_by_note_pk = itertools.groupby(self.attachmenets, key=lambda x: x.note_pk) - # note_pk_to_attachments_map = {k: list(v) for k, v in grouped_by_note_pk} - # - # for note_pk, note_attachments in note_pk_to_attachments_map.items(): grouped_by_parent_attachment_pk = itertools.groupby(self.attachments, key=lambda x: x.parent_pk) parent_pk_to_attachments_map = {k: list(v) for k, v in grouped_by_parent_attachment_pk if k is not None} @@ -243,7 +253,7 @@ def list_all_notes(self) -> List[Note]: z_pk, uuid, url_scheme, title, folder, modified_at, preview, account, locked, pinned, gzip_content ) in note_rows: - # if uuid != 'A2A24A7A-8C56-4904-AE68-6EAF466B1178': + # if uuid != '43EBA8F2-6FEA-470B-B943-4314448A0C6B': # continue utc_modified_at = datetime.strptime(modified_at, "%Y-%m-%d %H:%M:%S") @@ -265,10 +275,8 @@ def build_doc_content(self, compressed_data: bytes) -> NoteContent: note_raw = store.document.note attribute_text_list = self.build_attribute_text(note_raw.note_text, note_raw.attribute_run) - # for attribute_text in attribute_text_list: - # print(attribute_text) - - doc_paragraph_list = self.build_paragraph_list(attribute_text_list) + doc_content_lines: List[NoteContentLine] = build_content_lines(attribute_text_list) + doc_paragraph_list = build_paragraph_list(doc_content_lines) content = NoteContent(plan_text = note_raw.note_text, attributed_text = attribute_text_list, paragraph_list = doc_paragraph_list) return content @@ -283,9 +291,16 @@ def build_attribute_text(self, plan_text: str, attribute_run_list: Sequence[Attr for attribute_run in attribute_run_list: length = attribute_run.length * 2 # utf-16 length + # text and attribute text = utf16_text[cur_idx: cur_idx + length].decode("utf-16le") - # print(text.__repr__(), attribute_run) - attribute = self.build_note_attribute(attribute_run) + attribute = build_note_attribute(attribute_run) + + # logger.debug(f"attribute before: ({text_format.MessageToString(attribute_run, as_one_line=True)})") + # logger.debug(f"attribute after : {attribute.__repr__()}") + + # attachment + if identifier := attribute.attachment_identifier: + attribute.attachment = self.attachment_map_by_identifier.get(identifier) # notice: we convert apple note length to utf-8 length node = AttributeText(start_index = cur_idx, length = len(text), text = text, attribute = attribute) @@ -295,114 +310,4 @@ def build_attribute_text(self, plan_text: str, attribute_run_list: Sequence[Attr assert cur_idx == len(utf16_text) return struct_list - - def build_note_paragraph_style(self, paragraph_style: ParagraphStyle_pb2) -> ParagraphStyle: - style_type: None | ParagraphStyleType = None - indent_amount = 0 - check_info: None | CheckInfo = None - is_quote: bool = False - - if paragraph_style.HasField("checklist"): - check_info = self.build_check(paragraph_style.checklist) - if paragraph_style.HasField("style_type"): - style_type = ParagraphStyleType(paragraph_style.style_type) - if paragraph_style.HasField("indent_amount"): - indent_amount = paragraph_style.indent_amount - if paragraph_style.HasField("block_quote"): - is_quote = paragraph_style.block_quote == 1 - - return ParagraphStyle(style_type = style_type, indent_level = indent_amount, check_info = check_info, quote = is_quote) - - def build_note_attribute(self, attribute_run: AttributeRun) -> TextAttribute: - - paragraph_style: None | ParagraphStyle = None - font_style: None | FontStyle = None - font: None | str = None - underlined: None | bool = None - strike_through: None | bool = None - attachment: None | NoteAttachment = None - link: None | str = None - - if attribute_run.HasField("paragraph_style"): - paragraph_style = self.build_note_paragraph_style(attribute_run.paragraph_style) - if attribute_run.HasField("font"): - font = attribute_run.font.font_name - if attribute_run.HasField("font_weight"): - font_style = FontStyle(attribute_run.font_weight) - if attribute_run.HasField("underlined"): - underlined = attribute_run.underlined == 1 - if attribute_run.HasField("strikethrough"): - strike_through = attribute_run.strikethrough == 1 - if attribute_run.HasField("attachment_info"): - info = attribute_run.attachment_info - attachment = self.attachment_by_identify_map.get(info.attachment_identifier) - if attribute_run.HasField("link"): - link = attribute_run.link - - return TextAttribute( - paragraph_style = paragraph_style, - font_style = font_style, - font_name = font, - underlined = underlined, - strike_through = strike_through, - link = link, - attachment = attachment, - ) - - @classmethod - def build_check(cls, check: Checklist) -> None | CheckInfo: - if check.uuid == b'': - return None - return CheckInfo(done = (check.done == 1), uuid = check.uuid.hex()) - - @staticmethod - def build_paragraph_list(attribute_text_list: Sequence[AttributeText]) -> Sequence[NoteContentParagraph]: - """ - build content structure from attribute texts. - 1. build lines - 2. build paragraphs - """ - - split_list = [attribute.splitlines() for attribute in attribute_text_list] - flatten_list = itertools.chain.from_iterable(split_list) - - # build a line. - line_idx = 0 - lines: List[NoteContentLine] = [] - - one_line_stack: List[AttributeText] = [] - for text_attribute in flatten_list: - if text_attribute is None: - continue - one_line_stack.append(text_attribute) - if "\n" not in text_attribute.text: - continue - - # building - line = NoteContentLine.of(idx = line_idx, elements = one_line_stack) - - lines.append(line) - line_idx += 1 - one_line_stack = [] - - # build paragraph - paragraphs: List[NoteContentParagraph] = [] - - paragraph_stack = [] - previous_line = None - for line in lines: - paragraph_stack.append(line) - - if ( - not (previous_line is not None and not line.is_same_paragraph(previous_line)) - and not line.is_paragraph_breaker() - ): - previous_line = line - else: - attribute: ParagraphStyle | None = None if len(paragraph_stack) == 0 else paragraph_stack[0].attribute - paragraph = NoteContentParagraph(attribute = attribute, lines = paragraph_stack) - paragraphs.append(paragraph) - paragraph_stack = [] - previous_line = None - - return paragraphs + diff --git a/packages/pkg-core/loci/infra/fetchers/base.py b/packages/pkg-core/koma/infra/fetchers/base.py similarity index 78% rename from packages/pkg-core/loci/infra/fetchers/base.py rename to packages/pkg-core/koma/infra/fetchers/base.py index a61ab5e..d50a2b6 100644 --- a/packages/pkg-core/loci/infra/fetchers/base.py +++ b/packages/pkg-core/koma/infra/fetchers/base.py @@ -16,7 +16,7 @@ def __init__(self, renderer: Renderer): atexit.register(self.release) def shutdown(self): - raise NotImplementedError + pass def release(self): self.shutdown() @@ -25,8 +25,10 @@ def start_fetch(self) -> List[Note]: raise NotImplementedError def start_render(self): - for memory in self.notes: - self.renderer.start_pipline(memory) + for note in self.notes: + self.renderer.pre_render(note) + note.render(self.renderer) + self.renderer.post_render(note) def finish(self): pass diff --git a/packages/pkg-core/koma/infra/helper/__init__.py b/packages/pkg-core/koma/infra/helper/__init__.py new file mode 100644 index 0000000..8a0337f --- /dev/null +++ b/packages/pkg-core/koma/infra/helper/__init__.py @@ -0,0 +1,4 @@ + +from .apple import AppleNotesTableConstructor, build_paragraph_list, build_note_attribute, build_content_lines + +__all__ = ['AppleNotesTableConstructor', 'build_paragraph_list', 'build_note_attribute', 'build_content_lines'] \ No newline at end of file diff --git a/packages/pkg-core/loci/infra/helper/apple.py b/packages/pkg-core/koma/infra/helper/apple.py similarity index 56% rename from packages/pkg-core/loci/infra/helper/apple.py rename to packages/pkg-core/koma/infra/helper/apple.py index 744cfe8..2023a0a 100644 --- a/packages/pkg-core/loci/infra/helper/apple.py +++ b/packages/pkg-core/koma/infra/helper/apple.py @@ -1,15 +1,149 @@ -# ----------------------------------------------------------------------- -# Thanks to the source code from https://github.com/threeplanetssoftware/apple_cloud_notes_parser/blob/master/lib/AppleNotesEmbeddedTable.rb -# This file was generated by OpenAI's GPT-4 -# ----------------------------------------------------------------------- import itertools -from typing import List, Dict - -from ...domain import NoteAttachmentTableCell -from ...protobuf import MergableDataProto, MergeableDataObjectEntry, MergeableDataObjectMap, DictionaryElement +from typing import Dict, List, Sequence + +from ...domain import ( + AttributeText, + CheckInfo, + FontStyle, + NoteAttachmentTableCell, + NoteContentLine, + NoteContentParagraph, + ParagraphStyle, + ParagraphStyleType, + TextAttribute, +) +from ...protobuf import ( + AttributeRun, + Checklist, + DictionaryElement, + MergableDataProto, + MergeableDataObjectEntry, + MergeableDataObjectMap, + ParagraphStyle_pb2, +) + + +def build_check(check: Checklist) -> None | CheckInfo: + if check.uuid == b'': + return None + return CheckInfo(done = (check.done == 1), uuid = check.uuid.hex()) + + +def build_note_attribute(attribute_run: AttributeRun) -> TextAttribute: + + paragraph_style: None | ParagraphStyle = None + font_style: None | FontStyle = None + font: None | str = None + underlined: None | bool = None + strike_through: None | bool = None + attachment_identifier: None | str = None + link: None | str = None + + if attribute_run.HasField("paragraph_style"): + paragraph_style = build_note_paragraph_style(attribute_run.paragraph_style) + if attribute_run.HasField("font"): + font = attribute_run.font.font_name + if attribute_run.HasField("font_weight"): + font_style = FontStyle(attribute_run.font_weight) + if attribute_run.HasField("underlined"): + underlined = attribute_run.underlined == 1 + if attribute_run.HasField("strikethrough"): + strike_through = attribute_run.strikethrough == 1 + if attribute_run.HasField("attachment_info"): + info = attribute_run.attachment_info + attachment_identifier = info.attachment_identifier + if attribute_run.HasField("link"): + link = attribute_run.link + + return TextAttribute( + paragraph_style = paragraph_style, + font_style = font_style, + font_name = font, + underlined = underlined, + strike_through = strike_through, + link = link, + attachment_identifier = attachment_identifier, + ) + + +def build_note_paragraph_style(paragraph_style: ParagraphStyle_pb2) -> ParagraphStyle: + style_type: None | ParagraphStyleType = None + indent_amount = 0 + check_info: None | CheckInfo = None + is_quote: bool = False + + if paragraph_style.HasField("checklist"): + check_info = build_check(paragraph_style.checklist) + if paragraph_style.HasField("style_type"): + style_type = ParagraphStyleType(paragraph_style.style_type) + if paragraph_style.HasField("indent_amount"): + indent_amount = paragraph_style.indent_amount + if paragraph_style.HasField("block_quote"): + is_quote = paragraph_style.block_quote == 1 + + return ParagraphStyle(style_type = style_type, indent_level = indent_amount, check_info = check_info, quote = is_quote) + + +def build_content_lines(attribute_text_list: Sequence[AttributeText]) -> List[NoteContentLine]: + split_list = [attribute.splitlines() for attribute in attribute_text_list] + flatten_text_list = list(itertools.chain.from_iterable(split_list)) + + # build a line. + lines: List[NoteContentLine] = [] + + line_idx = 0 + one_line_stack: List[AttributeText] = [] + for idx, text_attribute in enumerate(flatten_text_list): + if text_attribute is None: + continue + one_line_stack.append(text_attribute) + if "\n" not in text_attribute.text and idx != len(flatten_text_list) - 1: + continue + + # building + line = NoteContentLine.of(idx = line_idx, elements = one_line_stack) + + lines.append(line) + line_idx += 1 + one_line_stack = [] + return lines + + +def build_paragraph_list(content_lines: List[NoteContentLine]) -> Sequence[NoteContentParagraph]: + + # build paragraph + paragraphs: List[NoteContentParagraph] = [] + + paragraph_stack = [] + previous_line = None + for idx, line in enumerate(content_lines): + paragraph_stack.append(line) + + if ( + not (previous_line is not None and not line.is_same_paragraph(previous_line)) + and not line.is_paragraph_breaker() + and idx != len(content_lines) - 1 + ): + previous_line = line + else: + attribute: ParagraphStyle | None = None if len(paragraph_stack) == 0 else paragraph_stack[0].attribute + paragraph = NoteContentParagraph(attribute = attribute, lines = paragraph_stack) + paragraphs.append(paragraph) + paragraph_stack = [] + previous_line = None + + # for paragraph in paragraphs: + # logger.debug(f"{paragraph.__repr__()}") + + return paragraphs class AppleNotesTableConstructor: + """ + Thanks to the source code from [AppleNotesEmbeddedTable.rb](https://github.com/threeplanetssoftware/apple_cloud_notes_parser/blob/master/lib/AppleNotesEmbeddedTable.rb) + This class was generated by OpenAI's GPT-4 + """ + LEFT_TO_RIGHT_DIRECTION = "CRTableColumnDirectionLeftToRight" RIGHT_TO_LEFT_DIRECTION = "CRTableColumnDirectionRightToLeft" @@ -25,7 +159,6 @@ class AppleNotesTableConstructor: def __init__(self, mergable_data_proto: MergableDataProto): self.key_items: List[str] = [] self.type_items: List[str] = [] - # self.row_items = [] self.table_objects: List[MergeableDataObjectEntry] = [] self.uuid_items: List[bytes] = [] self.total_columns = 0 diff --git a/packages/pkg-core/loci/infra/renderers/__init__.py b/packages/pkg-core/koma/infra/renderers/__init__.py similarity index 100% rename from packages/pkg-core/loci/infra/renderers/__init__.py rename to packages/pkg-core/koma/infra/renderers/__init__.py diff --git a/packages/pkg-core/loci/infra/renderers/base.py b/packages/pkg-core/koma/infra/renderers/base.py similarity index 88% rename from packages/pkg-core/loci/infra/renderers/base.py rename to packages/pkg-core/koma/infra/renderers/base.py index e5c192b..1a89994 100644 --- a/packages/pkg-core/loci/infra/renderers/base.py +++ b/packages/pkg-core/koma/infra/renderers/base.py @@ -3,10 +3,16 @@ # if TYPE_CHECKING: from ...domain import Note, NoteContent, NoteContentParagraph, NoteContentLine, AttributeText, NoteAttachmentLink, NoteAttachmentMedia, NoteAttachmentTag, NoteAttachmentDraw, NoteAttachmentTable, NoteAttachmentGallery -from ...core import RenderAble, BaseRenderer +from ...core import RenderAble, TextRenderer, Model -class Renderer(BaseRenderer, metaclass=ABCMeta): +class Renderer(TextRenderer, Model, metaclass=ABCMeta): + + def pre_render(self, render_able: RenderAble): + pass + + def post_render(self, render_able: RenderAble): + pass def render_attachment_link(self, attachment: NoteAttachmentLink) -> str: return "" @@ -28,23 +34,23 @@ def render_attachment_gallery(self, attribute: NoteAttachmentGallery) -> str: @abstractmethod def render_attribute_text(self, attribute: AttributeText) -> str: - raise NotImplementedError + ... @abstractmethod def render_line(self, line: NoteContentLine) -> str: - raise NotImplementedError + ... @abstractmethod def render_paragraph(self, paragraph: NoteContentParagraph) -> str: - raise NotImplementedError + ... @abstractmethod def render_content(self, content: NoteContent) -> str: - raise NotImplementedError + ... @abstractmethod def render_memory(self, memory: Note) -> str: - raise NotImplementedError + ... def render(self, render_able: RenderAble): if isinstance(render_able, Note): @@ -72,6 +78,3 @@ def render(self, render_able: RenderAble): else: raise RuntimeError(f"Unknown type {type(self)}") - def start_pipline(self, memory: RenderAble): - memory.render(self) - diff --git a/packages/pkg-core/loci/infra/renderers/markdown.py b/packages/pkg-core/koma/infra/renderers/markdown.py similarity index 86% rename from packages/pkg-core/loci/infra/renderers/markdown.py rename to packages/pkg-core/koma/infra/renderers/markdown.py index 02695c4..b9fe314 100644 --- a/packages/pkg-core/loci/infra/renderers/markdown.py +++ b/packages/pkg-core/koma/infra/renderers/markdown.py @@ -1,10 +1,17 @@ +import logging + +from ...core.render import RenderAble from ..renderers.base import Renderer from ...domain import Note, NoteContent, NoteContentParagraph, NoteContentLine, AttributeText, NoteAttachmentLink, NoteAttachmentMedia, NoteAttachmentTag, NoteAttachmentDraw, NoteAttachmentTable, NoteAttachmentGallery, ParagraphStyleType, FontStyle +logger = logging.getLogger(__name__) + class MarkDown(Renderer): - media_root_path: str + def post_render(self, render_able: RenderAble): + # logger.debug(f"Rendered: {render_able.represent.__repr__()}") + pass def render_attachment_link(self, attachment: NoteAttachmentLink) -> str: return f"[{attachment.text}]({attachment.url})" @@ -15,11 +22,11 @@ def render_attachment_media(self, attachment: NoteAttachmentMedia) -> str: if text is None and media_path is not None: return "" elif text is not None and media_path is None: - return f"[{text}]()" + return f"![{text}]()" elif text is None and media_path is not None: - return f"[]({media_path})" + return f"![]({media_path})" else: - return f"[{text}]({media_path})" + return f"![{text}]({media_path})" def render_attachment_tag(self, attachment: NoteAttachmentTag) -> str: return attachment.text if attachment.text is not None else "" @@ -29,9 +36,11 @@ def render_attachment_draw(self, attachment: NoteAttachmentDraw) -> str: return f"[{attachment.text}]({media_path})" def render_attachment_table(self, attribute: NoteAttachmentTable) -> str: + # TODO: Implement table rendering return "" def render_attachment_gallery(self, attribute: NoteAttachmentGallery) -> str: + # TODO: Implement gallery rendering return "" def render_attribute_text(self, attribute_text: AttributeText) -> str: @@ -41,7 +50,7 @@ def render_attribute_text(self, attribute_text: AttributeText) -> str: return attribute_text.text if attribute.attachment: - represent = attribute.attachment.represent + represent = attribute.attachment.rendered_result return represent if represent is not None else "" markdown_text = attribute_text.text @@ -66,9 +75,9 @@ def render_attribute_text(self, attribute_text: AttributeText) -> str: def render_line(self, line: NoteContentLine) -> str: markdown_text = "".join([ - ele.represent + ele.rendered_result for ele in line.elements - if ele.represent is not None + if ele.rendered_result is not None ]) return markdown_text @@ -77,9 +86,9 @@ def render_paragraph(self, paragraph: NoteContentParagraph) -> str: attribute = paragraph.attribute markdown_text = "".join([ - line.represent + line.rendered_result for line in paragraph.lines - if line.represent is not None + if line.rendered_result is not None ]) if attribute is None: @@ -96,7 +105,7 @@ def render_paragraph(self, paragraph: NoteContentParagraph) -> str: # if line.is_paragraph_breaker(): # continue - markdown_line = line.represent + markdown_line = line.rendered_result line_attribute = line.attribute if markdown_line is None: @@ -145,16 +154,16 @@ def render_paragraph(self, paragraph: NoteContentParagraph) -> str: else: markdown_text += markdown_line + logger.debug(f"paragraph: {markdown_text.__repr__()}") return markdown_text def render_content(self, content: NoteContent) -> str: return "".join([ - paragraph.represent + paragraph.rendered_result for paragraph in content.paragraph_list - if paragraph.represent is not None + if paragraph.rendered_result is not None ]) def render_memory(self, memory: Note) -> str: - re = memory.content.represent - # print(re) + re = memory.content.rendered_result return re if re is not None else "" diff --git a/packages/pkg-core/loci/protobuf/__init__.py b/packages/pkg-core/koma/protobuf/__init__.py similarity index 100% rename from packages/pkg-core/loci/protobuf/__init__.py rename to packages/pkg-core/koma/protobuf/__init__.py diff --git a/packages/pkg-core/loci/protobuf/notestore_pb2.py b/packages/pkg-core/koma/protobuf/notestore_pb2.py similarity index 100% rename from packages/pkg-core/loci/protobuf/notestore_pb2.py rename to packages/pkg-core/koma/protobuf/notestore_pb2.py diff --git a/packages/pkg-core/loci/protobuf/notestore_pb2.pyi b/packages/pkg-core/koma/protobuf/notestore_pb2.pyi similarity index 100% rename from packages/pkg-core/loci/protobuf/notestore_pb2.pyi rename to packages/pkg-core/koma/protobuf/notestore_pb2.pyi diff --git a/packages/pkg-core/loci/core/__init__.py b/packages/pkg-core/loci/core/__init__.py deleted file mode 100644 index 6262768..0000000 --- a/packages/pkg-core/loci/core/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .render import RenderAble, BaseRenderer - -__all__ = [ - 'BaseRenderer', 'RenderAble' -] \ No newline at end of file diff --git a/packages/pkg-core/loci/infra/helper/__init__.py b/packages/pkg-core/loci/infra/helper/__init__.py deleted file mode 100644 index 98012b6..0000000 --- a/packages/pkg-core/loci/infra/helper/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ - -from .apple import AppleNotesTableConstructor - -__all__ = ['AppleNotesTableConstructor'] \ No newline at end of file diff --git a/packages/pkg-core/pdm.lock b/packages/pkg-core/pdm.lock index 60b42e0..b35c529 100644 --- a/packages/pkg-core/pdm.lock +++ b/packages/pkg-core/pdm.lock @@ -5,161 +5,100 @@ groups = ["default"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:3a3dba57b04e218eaf1fcbdfde449e9ce92ecffc2c59269963982e5caedb935d" +content_hash = "sha256:b8463adbcaa15e423eca35a826fcf773189a6e708adfb941318cc41f52b84d1d" [[package]] -name = "greenlet" -version = "3.0.3" -requires_python = ">=3.7" -summary = "Lightweight in-process concurrent programming" +name = "annotated-types" +version = "0.7.0" +requires_python = ">=3.8" +summary = "Reusable constraint types to use with typing.Annotated" groups = ["default"] files = [ - {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, - {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, - {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] [[package]] name = "protobuf" -version = "5.26.1" +version = "5.27.0" requires_python = ">=3.8" summary = "" groups = ["default"] files = [ - {file = "protobuf-5.26.1-cp310-abi3-win32.whl", hash = "sha256:3c388ea6ddfe735f8cf69e3f7dc7611e73107b60bdfcf5d0f024c3ccd3794e23"}, - {file = "protobuf-5.26.1-cp310-abi3-win_amd64.whl", hash = "sha256:e6039957449cb918f331d32ffafa8eb9255769c96aa0560d9a5bf0b4e00a2a33"}, - {file = "protobuf-5.26.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:38aa5f535721d5bb99861166c445c4105c4e285c765fbb2ac10f116e32dcd46d"}, - {file = "protobuf-5.26.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fbfe61e7ee8c1860855696e3ac6cfd1b01af5498facc6834fcc345c9684fb2ca"}, - {file = "protobuf-5.26.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f7417703f841167e5a27d48be13389d52ad705ec09eade63dfc3180a959215d7"}, - {file = "protobuf-5.26.1-py3-none-any.whl", hash = "sha256:da612f2720c0183417194eeaa2523215c4fcc1a1949772dc65f05047e08d5932"}, - {file = "protobuf-5.26.1.tar.gz", hash = "sha256:8ca2a1d97c290ec7b16e4e5dff2e5ae150cc1582f55b5ab300d45cb0dfa90e51"}, -] - -[[package]] -name = "psycopg" -version = "3.1.19" -requires_python = ">=3.7" -summary = "PostgreSQL database adapter for Python" -groups = ["default"] -dependencies = [ - "typing-extensions>=4.1", - "tzdata; sys_platform == \"win32\"", -] -files = [ - {file = "psycopg-3.1.19-py3-none-any.whl", hash = "sha256:dca5e5521c859f6606686432ae1c94e8766d29cc91f2ee595378c510cc5b0731"}, - {file = "psycopg-3.1.19.tar.gz", hash = "sha256:92d7b78ad82426cdcf1a0440678209faa890c6e1721361c2f8901f0dccd62961"}, -] - -[[package]] -name = "psycopg-binary" -version = "3.1.19" -requires_python = ">=3.7" -summary = "PostgreSQL database adapter for Python -- C optimisation distribution" -groups = ["default"] -marker = "implementation_name != \"pypy\"" -files = [ - {file = "psycopg_binary-3.1.19-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4aa0ca13bb8a725bb6d12c13999217fd5bc8b86a12589f28a74b93e076fbb959"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:469424e354ebcec949aa6aa30e5a9edc352a899d9a68ad7a48f97df83cc914cf"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b04f5349313529ae1f1c42fe1aa0443faaf50fdf12d13866c2cc49683bfa53d0"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959feabddc7fffac89b054d6f23f3b3c62d7d3c90cd414a02e3747495597f150"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e9da624a6ca4bc5f7fa1f03f8485446b5b81d5787b6beea2b4f8d9dbef878ad7"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1823221a6b96e38b15686170d4fc5b36073efcb87cce7d3da660440b50077f6"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:866db42f986298f0cf15d805225eb8df2228bf19f7997d7f1cb5f388cbfc6a0f"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:738c34657305b5973af6dbb6711b07b179dfdd21196d60039ca30a74bafe9648"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb9758473200384a04374d0e0cac6f451218ff6945a024f65a1526802c34e56e"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0e991632777e217953ac960726158987da684086dd813ac85038c595e7382c91"}, - {file = "psycopg_binary-3.1.19-cp312-cp312-win_amd64.whl", hash = "sha256:1d87484dd42c8783c44a30400949efb3d81ef2487eaa7d64d1c54df90cf8b97a"}, + {file = "protobuf-5.27.0-cp310-abi3-win32.whl", hash = "sha256:2f83bf341d925650d550b8932b71763321d782529ac0eaf278f5242f513cc04e"}, + {file = "protobuf-5.27.0-cp310-abi3-win_amd64.whl", hash = "sha256:b276e3f477ea1eebff3c2e1515136cfcff5ac14519c45f9b4aa2f6a87ea627c4"}, + {file = "protobuf-5.27.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:744489f77c29174328d32f8921566fb0f7080a2f064c5137b9d6f4b790f9e0c1"}, + {file = "protobuf-5.27.0-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:f51f33d305e18646f03acfdb343aac15b8115235af98bc9f844bf9446573827b"}, + {file = "protobuf-5.27.0-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:56937f97ae0dcf4e220ff2abb1456c51a334144c9960b23597f044ce99c29c89"}, + {file = "protobuf-5.27.0-py3-none-any.whl", hash = "sha256:673ad60f1536b394b4fa0bcd3146a4130fcad85bfe3b60eaa86d6a0ace0fa374"}, + {file = "protobuf-5.27.0.tar.gz", hash = "sha256:07f2b9a15255e3cf3f137d884af7972407b556a7a220912b252f26dc3121e6bf"}, ] [[package]] -name = "psycopg" -version = "3.1.19" -extras = ["binary"] -requires_python = ">=3.7" -summary = "PostgreSQL database adapter for Python" -groups = ["default"] -dependencies = [ - "psycopg-binary==3.1.19; implementation_name != \"pypy\"", - "psycopg==3.1.19", -] -files = [ - {file = "psycopg-3.1.19-py3-none-any.whl", hash = "sha256:dca5e5521c859f6606686432ae1c94e8766d29cc91f2ee595378c510cc5b0731"}, - {file = "psycopg-3.1.19.tar.gz", hash = "sha256:92d7b78ad82426cdcf1a0440678209faa890c6e1721361c2f8901f0dccd62961"}, -] - -[[package]] -name = "sqlalchemy" -version = "2.0.30" -requires_python = ">=3.7" -summary = "Database Abstraction Library" +name = "pydantic" +version = "2.7.3" +requires_python = ">=3.8" +summary = "Data validation using Python type hints" groups = ["default"] dependencies = [ - "greenlet!=0.4.17; platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\"", - "typing-extensions>=4.6.0", + "annotated-types>=0.4.0", + "pydantic-core==2.18.4", + "typing-extensions>=4.6.1", ] files = [ - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5a79d65395ac5e6b0c2890935bad892eabb911c4aa8e8015067ddb37eea3d56c"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a5baf9267b752390252889f0c802ea13b52dfee5e369527da229189b8bd592e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cb5a646930c5123f8461f6468901573f334c2c63c795b9af350063a736d0134"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296230899df0b77dec4eb799bcea6fbe39a43707ce7bb166519c97b583cfcab3"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c62d401223f468eb4da32627bffc0c78ed516b03bb8a34a58be54d618b74d472"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3b69e934f0f2b677ec111b4d83f92dc1a3210a779f69bf905273192cf4ed433e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win32.whl", hash = "sha256:77d2edb1f54aff37e3318f611637171e8ec71472f1fdc7348b41dcb226f93d90"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win_amd64.whl", hash = "sha256:b6c7ec2b1f4969fc19b65b7059ed00497e25f54069407a8701091beb69e591a5"}, - {file = "SQLAlchemy-2.0.30-py3-none-any.whl", hash = "sha256:7108d569d3990c71e26a42f60474b4c02c8586c4681af5fd67e51a044fdea86a"}, - {file = "SQLAlchemy-2.0.30.tar.gz", hash = "sha256:2b1708916730f4830bc69d6f49d37f7698b5bd7530aca7f04f785f8849e95255"}, + {file = "pydantic-2.7.3-py3-none-any.whl", hash = "sha256:ea91b002777bf643bb20dd717c028ec43216b24a6001a280f83877fd2655d0b4"}, + {file = "pydantic-2.7.3.tar.gz", hash = "sha256:c46c76a40bb1296728d7a8b99aa73dd70a48c3510111ff290034f860c99c419e"}, ] [[package]] -name = "sqlalchemy" -version = "2.0.30" -extras = ["asyncio"] -requires_python = ">=3.7" -summary = "Database Abstraction Library" +name = "pydantic-core" +version = "2.18.4" +requires_python = ">=3.8" +summary = "Core functionality for Pydantic validation and serialization" groups = ["default"] dependencies = [ - "greenlet!=0.4.17", - "sqlalchemy==2.0.30", + "typing-extensions!=4.7.0,>=4.6.0", ] files = [ - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5a79d65395ac5e6b0c2890935bad892eabb911c4aa8e8015067ddb37eea3d56c"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a5baf9267b752390252889f0c802ea13b52dfee5e369527da229189b8bd592e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cb5a646930c5123f8461f6468901573f334c2c63c795b9af350063a736d0134"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:296230899df0b77dec4eb799bcea6fbe39a43707ce7bb166519c97b583cfcab3"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c62d401223f468eb4da32627bffc0c78ed516b03bb8a34a58be54d618b74d472"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3b69e934f0f2b677ec111b4d83f92dc1a3210a779f69bf905273192cf4ed433e"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win32.whl", hash = "sha256:77d2edb1f54aff37e3318f611637171e8ec71472f1fdc7348b41dcb226f93d90"}, - {file = "SQLAlchemy-2.0.30-cp312-cp312-win_amd64.whl", hash = "sha256:b6c7ec2b1f4969fc19b65b7059ed00497e25f54069407a8701091beb69e591a5"}, - {file = "SQLAlchemy-2.0.30-py3-none-any.whl", hash = "sha256:7108d569d3990c71e26a42f60474b4c02c8586c4681af5fd67e51a044fdea86a"}, - {file = "SQLAlchemy-2.0.30.tar.gz", hash = "sha256:2b1708916730f4830bc69d6f49d37f7698b5bd7530aca7f04f785f8849e95255"}, + {file = "pydantic_core-2.18.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2"}, + {file = "pydantic_core-2.18.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3"}, + {file = "pydantic_core-2.18.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9"}, + {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c"}, + {file = "pydantic_core-2.18.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8"}, + {file = "pydantic_core-2.18.4-cp312-none-win32.whl", hash = "sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07"}, + {file = "pydantic_core-2.18.4-cp312-none-win_amd64.whl", hash = "sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a"}, + {file = "pydantic_core-2.18.4-cp312-none-win_arm64.whl", hash = "sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d"}, + {file = "pydantic_core-2.18.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee"}, + {file = "pydantic_core-2.18.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9"}, + {file = "pydantic_core-2.18.4.tar.gz", hash = "sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864"}, ] [[package]] name = "typing-extensions" -version = "4.11.0" +version = "4.12.1" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" groups = ["default"] files = [ - {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, - {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, -] - -[[package]] -name = "tzdata" -version = "2024.1" -requires_python = ">=2" -summary = "Provider of IANA time zone data" -groups = ["default"] -marker = "sys_platform == \"win32\"" -files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, + {file = "typing_extensions-4.12.1-py3-none-any.whl", hash = "sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a"}, + {file = "typing_extensions-4.12.1.tar.gz", hash = "sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1"}, ] diff --git a/packages/pkg-core/pyproject.toml b/packages/pkg-core/pyproject.toml index 3403594..5d7bc0c 100644 --- a/packages/pkg-core/pyproject.toml +++ b/packages/pkg-core/pyproject.toml @@ -1,14 +1,13 @@ [project] -name = "loci" +name = "koma" version = "0.1.0" -description = "Default template for PDM package" +description = "The knowledge of macOS for AI." authors = [ {name = "AFuture", email = "afuture.d@outlook.com"}, ] dependencies = [ "protobuf>=5.26.1", - "sqlalchemy[asyncio]>=2.0.30", - "psycopg[binary]>=3.1.19", + "pydantic>=2.7.3", ] requires-python = ">=3.12" readme = "README.md" @@ -17,6 +16,4 @@ license = {text = "MIT"} [build-system] requires = ["pdm-backend"] build-backend = "pdm.backend" - -[project.scripts] -qa = "q:cli" +[tool.pdm] diff --git a/packages/pkg-core/tests/IntegrateTests.py b/packages/pkg-core/tests/IntegrateTests.py deleted file mode 100644 index 884a683..0000000 --- a/packages/pkg-core/tests/IntegrateTests.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest -from unittest import IsolatedAsyncioTestCase - - -class IntegrateTests(unittest.TestCase): - - def testMarkdown(self): - from loci.infra.renderers.markdown import MarkDown - from loci.infra.fetchers.apple import AppleNotesFetcher - - markdown = MarkDown() - fetcher = AppleNotesFetcher(markdown) - fetcher.start() - - print(fetcher.notes) \ No newline at end of file diff --git a/packages/pkg-core/tests/test_fetcher_apple.py b/packages/pkg-core/tests/test_fetcher_apple.py new file mode 100644 index 0000000..8a792a8 --- /dev/null +++ b/packages/pkg-core/tests/test_fetcher_apple.py @@ -0,0 +1,38 @@ +from typing import List, Sequence +from koma.domain.models.note import NoteContentLine +from koma.domain.models.style import AttributeText, TextAttribute, ParagraphStyle +import pytest +import unittest + +from koma.infra.helper.apple import build_content_lines, build_paragraph_list + + +class AppleFetcherTests(unittest.TestCase): + + def test_build_lines(self): + nodes: Sequence[AttributeText] = [ + AttributeText(start_index=0, length=5, text="Hello ", attribute=TextAttribute()), + AttributeText(start_index=5, length=5, text="World", attribute=TextAttribute()), + AttributeText(start_index=10, length=2, text="!\n", attribute=TextAttribute()), + AttributeText(start_index=12, length=1, text="\n", attribute=TextAttribute()), + AttributeText(start_index=13, length=7, text="xixixi\n", attribute=TextAttribute()), + AttributeText(start_index=20, length=2, text="ha", attribute=TextAttribute()), + AttributeText(start_index=22, length=3, text="hx", attribute=TextAttribute()), + ] + + lines: List[NoteContentLine] = build_content_lines(nodes) + assert lines[0].plain_text == "Hello World!\n" + assert lines[1].plain_text == "\n" + assert lines[2].plain_text == "xixixi\n" + assert lines[3].plain_text == "hahx" + + def test_build_paragraph(self): + # build_paragraph_list + lines: List[NoteContentLine] = [ + NoteContentLine(idx=1, plain_text="Hello World!\n", elements=[], attribute=ParagraphStyle()) + ] + build_paragraph_list(lines) + + + + \ No newline at end of file diff --git a/packages/pkg-core/tests/test_integrate.py b/packages/pkg-core/tests/test_integrate.py new file mode 100644 index 0000000..bb33d04 --- /dev/null +++ b/packages/pkg-core/tests/test_integrate.py @@ -0,0 +1,12 @@ +import unittest + + +class IntegrateTests(unittest.TestCase): + + def test_markdown(self): + from koma.infra.renderers.markdown import MarkDown + from koma.infra.fetchers.apple import AppleNotesFetcher + + markdown = MarkDown() + fetcher = AppleNotesFetcher(markdown) + fetcher.start() \ No newline at end of file diff --git a/packages/pkg-core/tests/test_render.py b/packages/pkg-core/tests/test_render.py new file mode 100644 index 0000000..a16c26b --- /dev/null +++ b/packages/pkg-core/tests/test_render.py @@ -0,0 +1,66 @@ +import pytest +import unittest +from koma.core import TextRenderer, RenderAble + + +class RenderTests(unittest.TestCase): + + def test_base_render_not_implement(self): + with pytest.raises(TypeError): + _ = TextRenderer() + + def test_base_render_implement(self): + class TestRenderer(TextRenderer): + def render(self, renderable: RenderAble): + return "fake result" + + render = TestRenderer() + result = render.render(None) + assert result == "fake result" + + def test_render_able(self): + class TestRenderer(TextRenderer): + def render(self, renderable: RenderAble): + return "fake result" + + class TestModel(RenderAble): + pass + + render = TestRenderer() + model = TestModel() + + result = render.render(model) + assert result == "fake result" + + def test_render_able_recursive(self): + + class TestInnerModel(RenderAble): + text: str = "inner" + + class TestModel(RenderAble): + inner: TestInnerModel = TestInnerModel() + list_inner: list[TestInnerModel] = [TestInnerModel(), TestInnerModel()] + dict_inner: dict[str, TestInnerModel] = {"a": TestInnerModel(), "b": TestInnerModel()} + + class TestRenderer(TextRenderer): + def render(self, renderable: RenderAble): + if isinstance(renderable, TestInnerModel): + return "inner result" + if isinstance(renderable, TestModel): + return "outer result" + return "fake result" + + render = TestRenderer() + model = TestModel() + + model.render(render) + + assert model.rendered_result == "outer result" + + assert model.inner.rendered_result == "inner result" + + for value in model.list_inner: + assert value.rendered_result == "inner result" + + for value in model.dict_inner.values(): + assert value.rendered_result == "inner result" diff --git a/packages/pkg-core/tests/test_render_markdown.py b/packages/pkg-core/tests/test_render_markdown.py new file mode 100644 index 0000000..80702ff --- /dev/null +++ b/packages/pkg-core/tests/test_render_markdown.py @@ -0,0 +1,155 @@ +from pathlib import Path +import unittest + +from koma.domain.models.attachment import NoteAttachment +from koma.domain.models.style import AttributeText, FontStyle, TextAttribute +from koma.infra.renderers.markdown import MarkDown + + +class RenderMarkdownTests(unittest.TestCase): + + markdown = MarkDown() + + def test_render_link(self): + + model = NoteAttachment( + type_uti="public.url", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None, + url="https://www.google.com" + ) + result = self.markdown.render_attachment_link(model) + assert result == "[Google](https://www.google.com)" + + def test_render_media(self): + + model_1 = NoteAttachment( + type_uti="public.any", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None, + media_root_path="fake_path", + generation="fake_generation", + file_uuid="fake_uuid", + file_name="fake_name", + ) + result_1 = self.markdown.render_attachment_media(model_1) + assert result_1 == "![Google](fake_path/Media/fake_uuid/fake_generation/fake_name)" + + model_2 = NoteAttachment( + type_uti="public.any", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None, + media_root_path="fake_path", + generation=None, + file_uuid="fake_uuid", + file_name="fake_name", + ) + result_2 = self.markdown.render_attachment_media(model_2) + assert result_2 == "![Google](fake_path/Media/fake_uuid/fake_name)" + + def test_render_tag(self): + + model = NoteAttachment( + type_uti="com.apple.notes.inlinetextattachment.hashtag", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="#Google", + parent_pk=None + ) + result = self.markdown.render_attachment_tag(model) + assert result == "#Google" + + def test_render_draw(self): + + model_1 = NoteAttachment( + type_uti="com.apple.drawing", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None, + media_root_path="fake_path", + generation="fake_generation", + file_uuid="fake_uuid", + file_name="fake_name", + ) + result_1: str = self.markdown.render_attachment_draw(model_1) + assert result_1 == "[Google](fake_path/FallbackImages/fake_identifier/fake_generation/FallbackImage.png)" + + model_2 = NoteAttachment( + type_uti="com.apple.drawing", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None, + media_root_path="fake_path", + generation=None, + file_uuid="fake_uuid", + file_name="fake_name", + ) + result_2 = self.markdown.render_attachment_draw(model_2) + assert result_2 == "[Google](fake_path/FallbackImages/fake_identifier.jpg)" + + def test_render_table(self): + + model = NoteAttachment( + type_uti="com.apple.notes.table", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None, + table_cell_list=[] + ) + result = self.markdown.render_attachment_table(model) + assert result == "" + + def test_render_gallery(self): + + model = NoteAttachment( + type_uti="com.apple.notes.gallery", + z_pk=1, + identifier="fake_identifier", + note_pk=1, + text="Google", + parent_pk=None + ) + result = self.markdown.render_attachment_gallery(model) + assert result == "" + + def test_render_text(self): + + text_1 = AttributeText(start_index=0,length=1,text="a",attribute=TextAttribute()) + result_1 = self.markdown.render_attribute_text(text_1) + assert result_1 == "a" + + text_2 = AttributeText(start_index=0,length=1,text="a",attribute=TextAttribute(font_style=FontStyle.Bold)) + result_2 = self.markdown.render_attribute_text(text_2) + assert result_2 == "**a**" + + text_3 = AttributeText(start_index=0,length=1,text="a",attribute=TextAttribute(font_style=FontStyle.Italic)) + result_3 = self.markdown.render_attribute_text(text_3) + assert result_3 == "*a*" + + text_4 = AttributeText(start_index=0,length=1,text="a",attribute=TextAttribute(font_style=FontStyle.Bold_Italic)) + result_4 = self.markdown.render_attribute_text(text_4) + assert result_4 == "***a***" + + text_5 = AttributeText(start_index=0,length=1,text="a",attribute=TextAttribute(font_style=FontStyle.Bold, link="https://www.google.com")) + result_5 = self.markdown.render_attribute_text(text_5) + assert result_5 == "[**a**](https://www.google.com)" + + text_6 = AttributeText(start_index=0,length=1,text="a",attribute=TextAttribute(font_style=FontStyle.Bold, strike_through=True)) + result_6 = self.markdown.render_attribute_text(text_6) + assert result_6 == "~~**a**~~" diff --git a/packages/pkg-core/tests/test_style.py b/packages/pkg-core/tests/test_style.py new file mode 100644 index 0000000..de795a9 --- /dev/null +++ b/packages/pkg-core/tests/test_style.py @@ -0,0 +1,131 @@ +import unittest +import pytest + +from koma.domain import ParagraphStyleType, ParagraphStyle, CheckInfo + + +class StyleTests(unittest.TestCase): + def test_paragraph_style(self): + assert ParagraphStyleType.Title.group_identifier() == 0 + assert ParagraphStyleType.Heading.group_identifier() == 1 + assert ParagraphStyleType.Subheading.group_identifier() == 2 + assert ParagraphStyleType.CodeBlock.group_identifier() == 4 + assert ParagraphStyleType.DotList.group_identifier() == 100 + assert ParagraphStyleType.DashList.group_identifier() == 100 + assert ParagraphStyleType.NumberList.group_identifier() == 100 + assert ParagraphStyleType.TodoList.group_identifier() == 100 + + def test_paragraph_is_same_paragraph(self): + empty = ParagraphStyle() + title = ParagraphStyle(style_type=ParagraphStyleType.Title) + heading = ParagraphStyle(style_type=ParagraphStyleType.Heading) + subheading = ParagraphStyle(style_type=ParagraphStyleType.Subheading) + code_block = ParagraphStyle(style_type=ParagraphStyleType.CodeBlock) + dot_list = ParagraphStyle(style_type=ParagraphStyleType.DotList) + dash_list = ParagraphStyle(style_type=ParagraphStyleType.DashList) + number_list = ParagraphStyle(style_type=ParagraphStyleType.NumberList) + todo_list = ParagraphStyle(style_type=ParagraphStyleType.TodoList) + + assert empty.is_same_block(empty) + assert empty.is_same_block(title) + assert empty.is_same_block(heading) + assert empty.is_same_block(subheading) + assert empty.is_same_block(code_block) + assert empty.is_same_block(dot_list) + assert empty.is_same_block(dash_list) + assert empty.is_same_block(number_list) + assert empty.is_same_block(todo_list) + + assert title.is_same_block(empty) + assert title.is_same_block(title) + assert not title.is_same_block(heading) + assert not title.is_same_block(subheading) + assert not title.is_same_block(code_block) + assert not title.is_same_block(dot_list) + assert not title.is_same_block(dash_list) + assert not title.is_same_block(number_list) + assert not title.is_same_block(todo_list) + + assert heading.is_same_block(empty) + assert heading.is_same_block(title) + assert heading.is_same_block(heading) + assert not heading.is_same_block(subheading) + assert not heading.is_same_block(code_block) + assert not heading.is_same_block(dot_list) + assert not heading.is_same_block(dash_list) + assert not heading.is_same_block(number_list) + assert not heading.is_same_block(todo_list) + + assert subheading.is_same_block(empty) + assert subheading.is_same_block(title) + assert subheading.is_same_block(heading) + assert subheading.is_same_block(subheading) + assert not subheading.is_same_block(code_block) + assert not subheading.is_same_block(dot_list) + assert not subheading.is_same_block(dash_list) + assert not subheading.is_same_block(number_list) + assert not subheading.is_same_block(todo_list) + + assert code_block.is_same_block(empty) + assert code_block.is_same_block(title) + assert code_block.is_same_block(heading) + assert code_block.is_same_block(subheading) + assert code_block.is_same_block(code_block) + assert not code_block.is_same_block(dot_list) + assert not code_block.is_same_block(dash_list) + assert not code_block.is_same_block(number_list) + assert not code_block.is_same_block(todo_list) + + assert dot_list.is_same_block(empty) + assert dot_list.is_same_block(title) + assert dot_list.is_same_block(heading) + assert dot_list.is_same_block(subheading) + assert dot_list.is_same_block(code_block) + assert dot_list.is_same_block(dot_list) + assert dot_list.is_same_block(dash_list) + assert dot_list.is_same_block(number_list) + assert dot_list.is_same_block(todo_list) + + assert dash_list.is_same_block(empty) + assert dash_list.is_same_block(title) + assert dash_list.is_same_block(heading) + assert dash_list.is_same_block(subheading) + assert dash_list.is_same_block(code_block) + assert dash_list.is_same_block(dot_list) + assert dash_list.is_same_block(dash_list) + assert dash_list.is_same_block(number_list) + assert dash_list.is_same_block(todo_list) + + assert number_list.is_same_block(empty) + assert number_list.is_same_block(title) + assert number_list.is_same_block(heading) + assert number_list.is_same_block(subheading) + assert number_list.is_same_block(code_block) + assert number_list.is_same_block(dot_list) + assert number_list.is_same_block(dash_list) + assert number_list.is_same_block(number_list) + assert number_list.is_same_block(todo_list) + + assert todo_list.is_same_block(empty) + assert todo_list.is_same_block(title) + assert todo_list.is_same_block(heading) + assert todo_list.is_same_block(subheading) + assert todo_list.is_same_block(code_block) + assert todo_list.is_same_block(dot_list) + assert todo_list.is_same_block(dash_list) + assert todo_list.is_same_block(number_list) + assert todo_list.is_same_block(todo_list) + + def test_quote_is_same_paragraph(self): + empty = ParagraphStyle(quote=False) + quote = ParagraphStyle(quote=True) + + assert not quote.is_same_block(empty) + assert quote.is_same_block(quote) + assert not empty.is_same_block(quote) + assert empty.is_same_block(empty) + + def test_check_info_is_same_paragraph(self): + check = ParagraphStyle(check_info=CheckInfo(done=False, uuid="uuid")) + + assert check.is_same_block(check) diff --git a/packages/pkg-server/agent/middleware.py b/packages/pkg-server/agent/middleware.py new file mode 100644 index 0000000..86eb0d5 --- /dev/null +++ b/packages/pkg-server/agent/middleware.py @@ -0,0 +1,50 @@ +from datetime import datetime +import logging + +from django.http.response import HttpResponse +from django.http.request import HttpRequest +from pydantic import BaseModel + + +class LoggingRecordCxt(BaseModel): + api: str + t: datetime + method: str | None + param: dict + +class LoggingRecordIn(BaseModel): + context: LoggingRecordCxt + headers: str + reqesut: str + +class LoggingRecordOut(BaseModel): + headers: str + response: str + t: datetime + +class LoggingRecord(BaseModel): + env: str + input: LoggingRecordIn + output: LoggingRecordOut + + +class RequestLoggingMiddleware: + def __init__(self, get_response): + self.get_response = get_response + self.logger = logging.getLogger(self.__class__.__qualname__) + self.logger.info("RequestLoggingMiddleware initialized") + + def __call__(self, request: HttpRequest): + + cxt = LoggingRecordCxt(api=request.path, t=datetime.now(), method=request.method, param={}) + input = LoggingRecordIn(context=cxt, headers=repr(request.headers), reqesut=request.body.decode()) + + # core logic + response: HttpResponse = self.get_response(request) + + output = LoggingRecordOut(headers=repr(response.serialize_headers()), response=response.content.decode(), t=datetime.now()) + record = LoggingRecord(env="", input=input, output=output) + + self.logger.info(record.model_dump_json()) + + return response \ No newline at end of file diff --git a/packages/pkg-server/agent/settings.py b/packages/pkg-server/agent/settings.py index 983406e..04a4238 100644 --- a/packages/pkg-server/agent/settings.py +++ b/packages/pkg-server/agent/settings.py @@ -12,6 +12,8 @@ from os import environ from pathlib import Path +SERVER_URL = environ.get("SERVER_URL", "http://127.0.0.1:8000") + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -23,15 +25,16 @@ SECRET_KEY = 'django-insecure-k+gu53zkf5v#5pawy)39c16qv_3phd%7dt1wf9e5z3l738-)@o' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False -# ALLOWED_HOSTS = [] +ALLOWED_HOSTS = [ + "*" +] # Application definition INSTALLED_APPS = [ - 'rest_framework', "rag.apps.RagConfig", 'django.contrib.contenttypes', 'django.contrib.staticfiles', @@ -39,6 +42,7 @@ ] MIDDLEWARE = [ + 'agent.middleware.RequestLoggingMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', diff --git a/packages/pkg-server/agent/urls.py b/packages/pkg-server/agent/urls.py index f9827d4..4798285 100644 --- a/packages/pkg-server/agent/urls.py +++ b/packages/pkg-server/agent/urls.py @@ -1,26 +1,29 @@ -""" -URL configuration for agent project. +import logging +import traceback -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/5.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" -from django.contrib import admin -from django.urls import path, include -from django.conf.urls.static import static +from django.urls import path +from ninja import NinjaAPI -from agent import settings +from .settings import SERVER_URL + +logger = logging.getLogger(__name__) + +api = NinjaAPI(servers = [{"url": SERVER_URL}]) + +api.add_router("/rag", "rag.routers.router") + +@api.exception_handler(Exception) +def service_unavailable(request, exception: Exception): + + exception_stack = "".join(traceback.format_exception(exception)) + logger.error(exception_stack) + + return api.create_response( + request, + {"err_msg": exception.__str__(), "code": 500, "success": False}, + status=200, + ) urlpatterns = [ - path("rag/", include("rag.urls")), - path('api-auth/', include('rest_framework.urls', namespace='rest_framework')) -] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + path("api/", api.urls), +] diff --git a/packages/pkg-server/pdm.lock b/packages/pkg-server/pdm.lock index aea1e00..c20f39e 100644 --- a/packages/pkg-server/pdm.lock +++ b/packages/pkg-server/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:156ae16700a599d56fd7a8f1e05687b1716789c2c3c3ceaee9af093f3e9565a5" +content_hash = "sha256:4647b1a9745284c92d9e283e37c130eb1b4d3c9076b33a4206ce38b3a68f090c" [[package]] name = "annotated-types" @@ -44,6 +44,38 @@ files = [ {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"}, ] +[[package]] +name = "boto3" +version = "1.34.120" +requires_python = ">=3.8" +summary = "The AWS SDK for Python" +groups = ["default"] +dependencies = [ + "botocore<1.35.0,>=1.34.120", + "jmespath<2.0.0,>=0.7.1", + "s3transfer<0.11.0,>=0.10.0", +] +files = [ + {file = "boto3-1.34.120-py3-none-any.whl", hash = "sha256:3c42bc309246a761413f6e152f307f009e80e7c9fd03dd9e6c0dc8ab8b3a8fc1"}, + {file = "boto3-1.34.120.tar.gz", hash = "sha256:38893db8269d25b72cc6fbab97633bfc863eefde5456847169d06149a16aa6e0"}, +] + +[[package]] +name = "botocore" +version = "1.34.120" +requires_python = ">=3.8" +summary = "Low-level, data-driven core of boto 3." +groups = ["default"] +dependencies = [ + "jmespath<2.0.0,>=0.7.1", + "python-dateutil<3.0.0,>=2.1", + "urllib3!=2.2.0,<3,>=1.25.4; python_version >= \"3.10\"", +] +files = [ + {file = "botocore-1.34.120-py3-none-any.whl", hash = "sha256:92bd739938078c7a0b110689a3eee21ecb3954d90653da013d9f98ef1165d6f7"}, + {file = "botocore-1.34.120.tar.gz", hash = "sha256:5cc0fca43cb2aad54917a394a001ac9ba774d21ad6a08828002d54b601776f78"}, +] + [[package]] name = "certifi" version = "2024.2.2" @@ -95,6 +127,28 @@ files = [ {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] +[[package]] +name = "cohere" +version = "5.5.4" +requires_python = "<4.0,>=3.8" +summary = "" +groups = ["default"] +dependencies = [ + "boto3<2.0.0,>=1.34.0", + "fastavro<2.0.0,>=1.9.4", + "httpx-sse<0.5.0,>=0.4.0", + "httpx>=0.21.2", + "pydantic>=1.9.2", + "requests<3.0.0,>=2.0.0", + "tokenizers<0.16,>=0.15", + "types-requests<3.0.0,>=2.0.0", + "typing-extensions>=4.0.0", +] +files = [ + {file = "cohere-5.5.4-py3-none-any.whl", hash = "sha256:8b692dcb5e86b554e5884168a7d2454951ce102fbd983e9053ec933e06bf02fa"}, + {file = "cohere-5.5.4.tar.gz", hash = "sha256:14acb2ccf272e958f79f9241ae972fd82e96b9b8ee9e6922a5687370761203ec"}, +] + [[package]] name = "colorama" version = "0.4.6" @@ -135,17 +189,56 @@ files = [ ] [[package]] -name = "djangorestframework" -version = "3.15.1" -requires_python = ">=3.6" -summary = "Web APIs for Django, made easy." +name = "django-ninja" +version = "1.1.0" +requires_python = ">=3.7" +summary = "Django Ninja - Fast Django REST framework" groups = ["default"] dependencies = [ - "django>=3.0", + "Django>=3.1", + "pydantic<3.0.0,>=2.0", +] +files = [ + {file = "django_ninja-1.1.0-py3-none-any.whl", hash = "sha256:6330c3497061d9fd1f43c1200f85c13aab7687110e2899f8304e5aa476c10b44"}, + {file = "django_ninja-1.1.0.tar.gz", hash = "sha256:87bff046416a2653ed2fbef1408e101292bf8170684821bac82accfd73bef059"}, ] + +[[package]] +name = "fastavro" +version = "1.9.4" +requires_python = ">=3.8" +summary = "Fast read/write of AVRO files" +groups = ["default"] +files = [ + {file = "fastavro-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:68912f2020e1b3d70557260b27dd85fb49a4fc6bfab18d384926127452c1da4c"}, + {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6925ce137cdd78e109abdb0bc33aad55de6c9f2d2d3036b65453128f2f5f5b92"}, + {file = "fastavro-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b928cd294e36e35516d0deb9e104b45be922ba06940794260a4e5dbed6c192a"}, + {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:90c9838bc4c991ffff5dd9d88a0cc0030f938b3fdf038cdf6babde144b920246"}, + {file = "fastavro-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:eca6e54da571b06a3c5a72dbb7212073f56c92a6fbfbf847b91c347510f8a426"}, + {file = "fastavro-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a4b02839ac261100cefca2e2ad04cdfedc556cb66b5ec735e0db428e74b399de"}, + {file = "fastavro-1.9.4.tar.gz", hash = "sha256:56b8363e360a1256c94562393dc7f8611f3baf2b3159f64fb2b9c6b87b14e876"}, +] + +[[package]] +name = "filelock" +version = "3.14.0" +requires_python = ">=3.8" +summary = "A platform independent file lock." +groups = ["default"] files = [ - {file = "djangorestframework-3.15.1-py3-none-any.whl", hash = "sha256:3ccc0475bce968608cf30d07fb17d8e52d1d7fc8bfe779c905463200750cbca6"}, - {file = "djangorestframework-3.15.1.tar.gz", hash = "sha256:f88fad74183dfc7144b2756d0d2ac716ea5b4c7c9840995ac3bfd8ec034333c1"}, + {file = "filelock-3.14.0-py3-none-any.whl", hash = "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f"}, + {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, +] + +[[package]] +name = "fsspec" +version = "2024.6.0" +requires_python = ">=3.8" +summary = "File-system specification" +groups = ["default"] +files = [ + {file = "fsspec-2024.6.0-py3-none-any.whl", hash = "sha256:58d7122eb8a1a46f7f13453187bfea4972d66bf01618d37366521b1998034cee"}, + {file = "fsspec-2024.6.0.tar.gz", hash = "sha256:f579960a56e6d8038a9efc8f9c77279ec12e6299aa86b0769a7e9c46b94527c2"}, ] [[package]] @@ -192,6 +285,37 @@ files = [ {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"}, ] +[[package]] +name = "httpx-sse" +version = "0.4.0" +requires_python = ">=3.8" +summary = "Consume Server-Sent Event (SSE) messages with HTTPX." +groups = ["default"] +files = [ + {file = "httpx-sse-0.4.0.tar.gz", hash = "sha256:1e81a3a3070ce322add1d3529ed42eb5f70817f45ed6ec915ab753f961139721"}, + {file = "httpx_sse-0.4.0-py3-none-any.whl", hash = "sha256:f329af6eae57eaa2bdfd962b42524764af68075ea87370a2de920af5341e318f"}, +] + +[[package]] +name = "huggingface-hub" +version = "0.23.3" +requires_python = ">=3.8.0" +summary = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" +groups = ["default"] +dependencies = [ + "filelock", + "fsspec>=2023.5.0", + "packaging>=20.9", + "pyyaml>=5.1", + "requests", + "tqdm>=4.42.1", + "typing-extensions>=3.7.4.3", +] +files = [ + {file = "huggingface_hub-0.23.3-py3-none-any.whl", hash = "sha256:22222c41223f1b7c209ae5511d2d82907325a0e3cdbce5f66949d43c598ff3bc"}, + {file = "huggingface_hub-0.23.3.tar.gz", hash = "sha256:1a1118a0b3dea3bab6c325d71be16f5ffe441d32f3ac7c348d6875911b694b5b"}, +] + [[package]] name = "idna" version = "3.7" @@ -203,6 +327,17 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "jmespath" +version = "1.0.1" +requires_python = ">=3.7" +summary = "JSON Matching Expressions" +groups = ["default"] +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "numpy" version = "1.26.4" @@ -244,6 +379,17 @@ files = [ {file = "openai-1.30.1.tar.gz", hash = "sha256:4f85190e577cba0b066e1950b8eb9b11d25bc7ebcc43a86b326ce1bfa564ec74"}, ] +[[package]] +name = "packaging" +version = "24.0" +requires_python = ">=3.7" +summary = "Core utilities for Python packages" +groups = ["default"] +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + [[package]] name = "pgvector" version = "0.2.5" @@ -367,6 +513,20 @@ files = [ {file = "pydantic_core-2.18.2.tar.gz", hash = "sha256:2e29d20810dfc3043ee13ac7d9e25105799817683348823f305ab3f349b9386e"}, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +summary = "Extensions to the standard Python datetime module" +groups = ["default"] +dependencies = [ + "six>=1.5", +] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + [[package]] name = "python-dotenv" version = "1.0.1" @@ -378,6 +538,23 @@ files = [ {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, ] +[[package]] +name = "pyyaml" +version = "6.0.1" +requires_python = ">=3.6" +summary = "YAML parser and emitter for Python" +groups = ["default"] +files = [ + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + [[package]] name = "regex" version = "2024.5.15" @@ -420,6 +597,31 @@ files = [ {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, ] +[[package]] +name = "s3transfer" +version = "0.10.1" +requires_python = ">= 3.8" +summary = "An Amazon S3 Transfer Manager" +groups = ["default"] +dependencies = [ + "botocore<2.0a.0,>=1.33.2", +] +files = [ + {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"}, + {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"}, +] + +[[package]] +name = "six" +version = "1.16.0" +requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Python 2 and 3 compatibility utilities" +groups = ["default"] +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -463,6 +665,68 @@ files = [ {file = "tiktoken-0.7.0.tar.gz", hash = "sha256:1077266e949c24e0291f6c350433c6f0971365ece2b173a23bc3b9f9defef6b6"}, ] +[[package]] +name = "tokenizers" +version = "0.15.2" +requires_python = ">=3.7" +summary = "" +groups = ["default"] +dependencies = [ + "huggingface-hub<1.0,>=0.16.4", +] +files = [ + {file = "tokenizers-0.15.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f86593c18d2e6248e72fb91c77d413a815153b8ea4e31f7cd443bdf28e467670"}, + {file = "tokenizers-0.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0774bccc6608eca23eb9d620196687c8b2360624619623cf4ba9dc9bd53e8b51"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d0222c5b7c9b26c0b4822a82f6a7011de0a9d3060e1da176f66274b70f846b98"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3835738be1de66624fff2f4f6f6684775da4e9c00bde053be7564cbf3545cc66"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0143e7d9dcd811855c1ce1ab9bf5d96d29bf5e528fd6c7824d0465741e8c10fd"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db35825f6d54215f6b6009a7ff3eedee0848c99a6271c870d2826fbbedf31a38"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f5e64b0389a2be47091d8cc53c87859783b837ea1a06edd9d8e04004df55a5c"}, + {file = "tokenizers-0.15.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e0480c452217edd35eca56fafe2029fb4d368b7c0475f8dfa3c5c9c400a7456"}, + {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a33ab881c8fe70474980577e033d0bc9a27b7ab8272896e500708b212995d834"}, + {file = "tokenizers-0.15.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a308a607ca9de2c64c1b9ba79ec9a403969715a1b8ba5f998a676826f1a7039d"}, + {file = "tokenizers-0.15.2-cp312-none-win32.whl", hash = "sha256:b8fcfa81bcb9447df582c5bc96a031e6df4da2a774b8080d4f02c0c16b42be0b"}, + {file = "tokenizers-0.15.2-cp312-none-win_amd64.whl", hash = "sha256:38d7ab43c6825abfc0b661d95f39c7f8af2449364f01d331f3b51c94dcff7221"}, + {file = "tokenizers-0.15.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:38bfb0204ff3246ca4d5e726e8cc8403bfc931090151e6eede54d0e0cf162ef0"}, + {file = "tokenizers-0.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9c861d35e8286a53e06e9e28d030b5a05bcbf5ac9d7229e561e53c352a85b1fc"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:936bf3842db5b2048eaa53dade907b1160f318e7c90c74bfab86f1e47720bdd6"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:620beacc3373277700d0e27718aa8b25f7b383eb8001fba94ee00aeea1459d89"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2735ecbbf37e52db4ea970e539fd2d450d213517b77745114f92867f3fc246eb"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:473c83c5e2359bb81b0b6fde870b41b2764fcdd36d997485e07e72cc3a62264a"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968fa1fb3c27398b28a4eca1cbd1e19355c4d3a6007f7398d48826bbe3a0f728"}, + {file = "tokenizers-0.15.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:865c60ae6eaebdde7da66191ee9b7db52e542ed8ee9d2c653b6d190a9351b980"}, + {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7c0d8b52664ab2d4a8d6686eb5effc68b78608a9008f086a122a7b2996befbab"}, + {file = "tokenizers-0.15.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f33dfbdec3784093a9aebb3680d1f91336c56d86cc70ddf88708251da1fe9064"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6a9b648a58281c4672212fab04e60648fde574877d0139cd4b4f93fe28ca8944"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:7c7d18b733be6bbca8a55084027f7be428c947ddf871c500ee603e375013ffba"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:13ca3611de8d9ddfbc4dc39ef54ab1d2d4aaa114ac8727dfdc6a6ec4be017378"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:237d1bf3361cf2e6463e6c140628e6406766e8b27274f5fcc62c747ae3c6f094"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a0fe1e49e60c664915e9fb6b0cb19bac082ab1f309188230e4b2920230edb3"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4e022fe65e99230b8fd89ebdfea138c24421f91c1a4f4781a8f5016fd5cdfb4d"}, + {file = "tokenizers-0.15.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d857be2df69763362ac699f8b251a8cd3fac9d21893de129bc788f8baaef2693"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:708bb3e4283177236309e698da5fcd0879ce8fd37457d7c266d16b550bcbbd18"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c35e09e9899b72a76e762f9854e8750213f67567787d45f37ce06daf57ca78"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1257f4394be0d3b00de8c9e840ca5601d0a4a8438361ce9c2b05c7d25f6057b"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02272fe48280e0293a04245ca5d919b2c94a48b408b55e858feae9618138aeda"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dc3ad9ebc76eabe8b1d7c04d38be884b8f9d60c0cdc09b0aa4e3bcf746de0388"}, + {file = "tokenizers-0.15.2-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:32e16bdeffa7c4f46bf2152172ca511808b952701d13e7c18833c0b73cb5c23f"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fb16ba563d59003028b678d2361a27f7e4ae0ab29c7a80690efa20d829c81fdb"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:2277c36d2d6cdb7876c274547921a42425b6810d38354327dd65a8009acf870c"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cf75d32e8d250781940d07f7eece253f2fe9ecdb1dc7ba6e3833fa17b82fcbc"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b3b31884dc8e9b21508bb76da80ebf7308fdb947a17affce815665d5c4d028"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b10122d8d8e30afb43bb1fe21a3619f62c3e2574bff2699cf8af8b0b6c5dc4a3"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d88b96ff0fe8e91f6ef01ba50b0d71db5017fa4e3b1d99681cec89a85faf7bf7"}, + {file = "tokenizers-0.15.2-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:37aaec5a52e959892870a7c47cef80c53797c0db9149d458460f4f31e2fb250e"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e2ea752f2b0fe96eb6e2f3adbbf4d72aaa1272079b0dfa1145507bd6a5d537e6"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4b19a808d8799fda23504a5cd31d2f58e6f52f140380082b352f877017d6342b"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:64c86e5e068ac8b19204419ed8ca90f9d25db20578f5881e337d203b314f4104"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de19c4dc503c612847edf833c82e9f73cd79926a384af9d801dcf93f110cea4e"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea09acd2fe3324174063d61ad620dec3bcf042b495515f27f638270a7d466e8b"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cf27fd43472e07b57cf420eee1e814549203d56de00b5af8659cb99885472f1f"}, + {file = "tokenizers-0.15.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7ca22bd897537a0080521445d91a58886c8c04084a6a19e6c78c586e0cfa92a5"}, + {file = "tokenizers-0.15.2.tar.gz", hash = "sha256:e6e9c6e019dd5484be5beafc775ae6c925f4c69a3487040ed09b45e13df2cb91"}, +] + [[package]] name = "tqdm" version = "4.66.4" @@ -477,6 +741,20 @@ files = [ {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"}, ] +[[package]] +name = "types-requests" +version = "2.32.0.20240602" +requires_python = ">=3.8" +summary = "Typing stubs for requests" +groups = ["default"] +dependencies = [ + "urllib3>=2", +] +files = [ + {file = "types-requests-2.32.0.20240602.tar.gz", hash = "sha256:3f98d7bbd0dd94ebd10ff43a7fbe20c3b8528acace6d8efafef0b6a184793f06"}, + {file = "types_requests-2.32.0.20240602-py3-none-any.whl", hash = "sha256:ed3946063ea9fbc6b5fc0c44fa279188bae42d582cb63760be6cb4b9d06c3de8"}, +] + [[package]] name = "typing-extensions" version = "4.11.0" diff --git a/packages/pkg-server/pyproject.toml b/packages/pkg-server/pyproject.toml index 90bc973..62f8784 100644 --- a/packages/pkg-server/pyproject.toml +++ b/packages/pkg-server/pyproject.toml @@ -9,10 +9,11 @@ dependencies = [ "django>=5.0.6", "psycopg[binary]>=3.1.19", "python-dotenv>=1.0.1", - "djangorestframework>=3.15.1", "OpenAI>=1.30.1", "pgvector>=0.2.5", "tiktoken>=0.7.0", + "django-ninja>=1.1.0", + "cohere>=5.5.4", ] requires-python = ">=3.12" readme = "README.md" @@ -26,7 +27,14 @@ build-backend = "pdm.backend" [tool.pdm] distribution = true +[tool.pdm.build] +excludes = ["./**/.git"] + [tool.pdm.dev-dependencies] dev = [ "uvicorn>=0.29.0", ] + + +[tool.pytest] +DJANGO_SETTINGS_MODULE="agent.settings" \ No newline at end of file diff --git a/packages/pkg-server/rag/admin.py b/packages/pkg-server/rag/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/packages/pkg-server/rag/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/packages/pkg-server/rag/domain/__init__.py b/packages/pkg-server/rag/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/pkg-server/rag/domain/enum.py b/packages/pkg-server/rag/domain/enum.py index e4ba5cb..1b13a1e 100644 --- a/packages/pkg-server/rag/domain/enum.py +++ b/packages/pkg-server/rag/domain/enum.py @@ -1,7 +1,19 @@ from django.db import models from django.utils.translation import gettext_lazy as _ + class MemoryType(models.TextChoices): NOTE = "NOTE", _("NOTE") REMINDER = "REMINDER", _("REMINDER") - PIC = "PIC", _("PIC") \ No newline at end of file + PIC = "PIC", _("PIC") + + +class IndexState(models.TextChoices): + NOT_STARTED = "NOT_STARTED", _("not started") + PROCESSING = "PROCESSING", _("processing") + INDEXED = "INDEXED", _("indexed") + + +class EmbedModel(models.TextChoices): + OPENAI_TEXT_EMBEDDING_3_LARGE = "OPENAI_TEXT_EMBEDDING_3_LARGE", _("openai text-embedding-3-large") + OPENAI_TEXT_EMBEDDING_3_SMALL = "OPENAI_TEXT_EMBEDDING_3_SMALL", _("openai text-embedding-3-small") diff --git a/packages/pkg-server/rag/domain/manager.py b/packages/pkg-server/rag/domain/manager.py new file mode 100644 index 0000000..947de99 --- /dev/null +++ b/packages/pkg-server/rag/domain/manager.py @@ -0,0 +1,53 @@ +from typing import List + +from django.db import models +from pgvector.django import CosineDistance + +from .models import Memory, Neuron, MemorySyncLog, NeuronIndexLog + +MODEL_MANAGER_ATTRIBUTE = 'objects' + + +class MemoryManager(models.Manager[Memory]): + + def __init__(self): + super().__init__() + self.contribute_to_class(Memory, MODEL_MANAGER_ATTRIBUTE) + + def list_all(self) -> List[Memory]: + return list(self.all()) + + def list_by_biz_ids(self, biz_ids: List[str]) -> List[Memory]: + query = self.filter(biz_id__in = biz_ids) + return list(query) + + +class MemorySyncLogManager(models.Manager[MemorySyncLog]): + def __init__(self): + super().__init__() + self.contribute_to_class(MemorySyncLog, MODEL_MANAGER_ATTRIBUTE) + + def list_by_biz_ids(self, biz_ids: List[str]) -> List[MemorySyncLog]: + # group by biz_id and find the max modified_at + query = self.filter(biz_id__in = biz_ids).distinct('biz_id').order_by('biz_id', '-biz_modified_at') + return list(query) + + +class NeuronIndexLogManager(models.Manager[NeuronIndexLog]): + def __init__(self): + super().__init__() + self.contribute_to_class(NeuronIndexLog, MODEL_MANAGER_ATTRIBUTE) + + +class NeuronManager(models.Manager[Neuron]): + + def __init__(self): + super().__init__() + self.contribute_to_class(Neuron, MODEL_MANAGER_ATTRIBUTE) + + def list_within_distance_on_embedding(self, embedding: List[float], distance: float) -> List[Neuron]: + query = self.alias(distance = CosineDistance('embedding', embedding)).filter(distance__lt = distance) + return list(query) + + def delete_by_memory_ids(self, memory_ids: List[int]): + self.filter(memory_id__in = memory_ids).delete() \ No newline at end of file diff --git a/packages/pkg-server/rag/domain/models.py b/packages/pkg-server/rag/domain/models.py index 2d4f886..d01b74b 100644 --- a/packages/pkg-server/rag/domain/models.py +++ b/packages/pkg-server/rag/domain/models.py @@ -4,13 +4,14 @@ from pgvector.django import HnswIndex, VectorField from pydantic import BaseModel -from loci.domain.models.note import Note -from .enum import MemoryType +from koma.domain.models.note import Note + +from .enum import EmbedModel, IndexState, MemoryType + class AppleNoteField(models.JSONField): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def from_db_value(self, value, expression, connection): """ @@ -29,7 +30,7 @@ def get_prep_value(self, value: Any) -> Any: return value if isinstance(value, BaseModel): - return value.model_dump(mode = 'json') + return value.model_dump(mode = 'json', exclude_none = True) else: return value @@ -45,24 +46,31 @@ def to_python(self, value: Any) -> None | Note: elif isinstance(value, dict): json_value = value else: - super().to_python(value) + return super().to_python(value) return Note(**json_value) class Memory(models.Model): + memory_id = models.AutoField(primary_key = True) memory_type = models.CharField(choices = MemoryType.choices) data = AppleNoteField(null=True) - biz_id = models.CharField(max_length = 100, null = True) + biz_id = models.CharField(max_length = 100) updated_at = models.DateTimeField(auto_now = True) created_at = models.DateTimeField(auto_now_add = True) class Meta: db_table = "memories" + constraints = [ + models.UniqueConstraint(fields = ['biz_id'], name = 'uk_biz_id') + ] + class MemorySyncLog(models.Model): + log_id = models.AutoField(primary_key = True) + biz_id = models.CharField(max_length = 100) biz_modified_at = models.DateTimeField(auto_now = True) @@ -74,9 +82,6 @@ class Meta: indexes = [ models.Index(fields = ['biz_modified_at'], name = "idx_biz_modified_at") ] - constraints = [ - models.UniqueConstraint(fields = ['biz_id'], name = 'uk_biz_id') - ] ordering = ["-biz_modified_at"] @@ -86,6 +91,7 @@ class Position(BaseModel): paragraph: int | None = None line: int | None = None + class PositionField(models.JSONField): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -101,7 +107,7 @@ def get_prep_value(self, value: Any) -> Any: return value if isinstance(value, BaseModel): - return value.model_dump(mode = 'json') + return value.model_dump(mode = 'json', exclude_none = True) else: return value @@ -120,12 +126,33 @@ def to_python(self, value: Any) -> None | Position: super().to_python(value) return Position(**json_value) + +class NeuronIndexLog(models.Model): + log_id = models.AutoField(primary_key = True) + + memory_id = models.IntegerField() + state = models.CharField(choices = IndexState.choices, default=IndexState.NOT_STARTED) + indexed_at = models.DateTimeField(null=True) + + updated_at = models.DateTimeField(auto_now = True) + created_at = models.DateTimeField(auto_now_add = True) + + class Meta: + db_table = 'neuron_index_logs' + indexes = [ + models.Index(fields = ['memory_id'], name = 'idx_memory_id') + ] + + class Neuron(models.Model): + neuron_id = models.AutoField(primary_key = True) + content = models.TextField(null = False) embedding = VectorField(dimensions = 1536) - biz_id = models.CharField(max_length = 100, null = True) + memory_id = models.IntegerField() position = PositionField(null = True) + embed_model = models.CharField(choices = EmbedModel.choices) class Meta: db_table = 'neurons' @@ -137,4 +164,7 @@ class Meta: ef_construction = 64, opclasses = ['vector_cosine_ops'] ) - ] \ No newline at end of file + ] + + def __str__(self): + return self.content \ No newline at end of file diff --git a/packages/pkg-server/rag/dto/__init__.py b/packages/pkg-server/rag/dto/__init__.py new file mode 100644 index 0000000..dc7322a --- /dev/null +++ b/packages/pkg-server/rag/dto/__init__.py @@ -0,0 +1,4 @@ +from .common import Result +from .memory import MemoryDTO, NeuronDTO + +__ALL__ = ["Result", "MemoryDTO", "NeuronDTO"] \ No newline at end of file diff --git a/packages/pkg-server/rag/dto/common.py b/packages/pkg-server/rag/dto/common.py new file mode 100644 index 0000000..405f093 --- /dev/null +++ b/packages/pkg-server/rag/dto/common.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Generic, TypeVar + +from ninja import Schema +from pydantic import BaseModel + +T = TypeVar("T") + + +class Result(Schema, BaseModel, Generic[T]): + """ + Notice: The inhert order matters. + """ + + code: int + success: bool + data: T | None = None + err: str | None = None + + @staticmethod + def with_data(data: T) -> Result[T]: + return Result[T](code=200, success=True, data=data) + + @staticmethod + def succ() -> Result[T]: + return Result[T](code=200, success=True) + + @staticmethod + def with_err(code: int, err_msg: str) -> Result[T]: + return Result[T](code=code, success=False, err=err_msg) diff --git a/packages/pkg-server/rag/dto/memory.py b/packages/pkg-server/rag/dto/memory.py index 52f53a8..543af6a 100644 --- a/packages/pkg-server/rag/dto/memory.py +++ b/packages/pkg-server/rag/dto/memory.py @@ -1,17 +1,17 @@ from datetime import datetime from typing import List -from pydantic import BaseModel +from pydantic import BaseModel, Field -from loci.domain.models.note import Note +from koma.domain.models.note import Note from ..domain.models import Memory, Neuron class MemoryDTO(BaseModel): - memory_type: str + memory_type: str = Field(title = "The type of the memory.") data: Note # temporary usage - updated_at: datetime - created_at: datetime + updated_at: datetime = Field(title = "The last updated time of the memory.") + created_at: datetime = Field(title = "The created time of the memory.") @staticmethod def from_model(model: Memory) -> 'MemoryDTO': @@ -28,12 +28,13 @@ def from_model_list(models: List[Memory]) -> List['MemoryDTO']: class NeuronDTO(BaseModel): - content: str + content: str = Field(title = "The content of the neuron") + memory_id: int = Field(title = "The id of the memory that the neuron belongs to.") @staticmethod def from_model(model: Neuron) -> 'NeuronDTO': return NeuronDTO( - content=model.content, + content=model.content, memory_id=model.memory_id ) @staticmethod diff --git a/packages/pkg-server/rag/infra/__init__.py b/packages/pkg-server/rag/infra/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/pkg-server/rag/infra/services.py b/packages/pkg-server/rag/infra/services.py new file mode 100644 index 0000000..229a963 --- /dev/null +++ b/packages/pkg-server/rag/infra/services.py @@ -0,0 +1,211 @@ +import itertools +import logging +from datetime import datetime +from operator import ne +from typing import List + +import cohere +import openai +import tiktoken + +from koma.domain.models.note import Note +from koma.infra.fetchers.apple import AppleNotesFetcher +from koma.infra.renderers.markdown import MarkDown + +from ..domain.enum import EmbedModel, IndexState, MemoryType +from ..domain.manager import ( + MemoryManager, + MemorySyncLogManager, + NeuronIndexLogManager, + NeuronManager, +) +from ..domain.models import Memory, MemorySyncLog, Neuron, NeuronIndexLog, Position +from ..dto.memory import MemoryDTO, NeuronDTO + +logger = logging.getLogger(__name__) +openai_client = openai.OpenAI() +cohere_client = cohere.Client() + +class MemorySerivce: + def sync_modified_memories(self) -> List[Memory]: + markdown = MarkDown() + fetcher = AppleNotesFetcher(markdown) + fetcher.start() + + notes = fetcher.notes + uuids = [note.uuid for note in notes] + + logs = MemorySyncLogManager().list_by_biz_ids(uuids) + last_modified_map_by_biz_id = {log.biz_id: log.biz_modified_at for log in logs} + + to_create_notes = [ + note + for note in notes + if note.uuid not in last_modified_map_by_biz_id.keys() + ] + + to_update_notes = [ + note + for note in notes + if note.uuid in last_modified_map_by_biz_id.keys() and note.modified_at > last_modified_map_by_biz_id.get(note.uuid) + ] + + to_update_biz_ids = list(map(lambda note: note.uuid, to_update_notes)) + to_update_memories = MemoryManager().list_by_biz_ids(to_update_biz_ids) + + + created_memories = [ + Memory(memory_type = MemoryType.NOTE, data = note, biz_id = note.uuid) + for note in to_create_notes + ] + + to_update_notes_map_by_biz_id = {note.uuid: note for note in to_update_notes} + for memory in to_update_memories: + note = to_update_notes_map_by_biz_id.get(memory.biz_id) + if note is None: + continue + + memory.data = note + + MemoryManager().bulk_create(created_memories) + MemoryManager().bulk_update(to_update_memories, fields=["data"], batch_size=20) + + memories = created_memories + to_update_memories + + sync_logs = [ + MemorySyncLog( + biz_id = memory.biz_id, + biz_modified_at = memory.data.modified_at + ) + for memory in memories + ] + + MemorySyncLogManager().bulk_create(sync_logs) + + return memories + + +class NeuronService: + def index_memories(self, memories: List[Memory]): + + indexLogs = [ + NeuronIndexLog(memory_id = memory.memory_id) + for memory in memories + ] + NeuronIndexLogManager().bulk_create(indexLogs) + + memory_ids = [memory.memory_id for memory in memories] + NeuronManager().delete_by_memory_ids(memory_ids) + + neurons_mapper = map(generate_neurons, memories) + neurons = list(itertools.chain.from_iterable(neurons_mapper)) + + NeuronManager().bulk_create(neurons) + + for indexLog in indexLogs: + indexLog.state = IndexState.INDEXED + indexLog.indexed_at = datetime.now() + + NeuronIndexLogManager().bulk_update(indexLogs, fields=["state", "indexed_at"], batch_size=20) + + return + + def query_similar(self, query: str, topk: int) -> List[Neuron]: + result = openai_client.embeddings.create(input = query, model = "text-embedding-3-small") + embedding = result.data[0].embedding + + neurons = NeuronManager().list_within_distance_on_embedding(embedding = embedding, distance = 0.80) + reranked = self.rerank_neurons(neurons=neurons,query=query,topk=topk) + return reranked + + @staticmethod + def rerank_neurons(neurons: List[Neuron], query: str, topk: int) -> List[Neuron]: + + neuron_map_by_idx = {idx: neuron for idx, neuron in enumerate(neurons)} + documents = [neuron.content for neuron in neurons] + rerank_result = cohere_client.rerank(query=query, documents=documents, model="rerank-multilingual-v2.0", top_n=topk).results + + rerank_result_idx = [item.index for item in rerank_result] + return [ + neuron_map_by_idx[idx] + for idx in rerank_result_idx + ] + + +class MemoryBizService: + + memory_service = MemorySerivce() + neuron_service = NeuronService() + + def list_all(self) -> List[MemoryDTO]: + memories = MemoryManager().list_all() + dto = MemoryDTO.from_model_list(memories) + return dto + + def sync_memories(self): + memories = self.memory_service.sync_modified_memories() + + if len(memories) == 0: + return + + self.neuron_service.index_memories(memories) + return + + +def generate_neurons(memory: Memory) -> List[Neuron]: + note: Note = memory.data + paragraph_list = note.content.paragraph_list + if len(paragraph_list) == 0: + return [] + + # check if represent is not blank + indexable_paragraph = [ + {"idx":idx, "content": p.rendered_result} + for idx, p in enumerate(paragraph_list) + if p.rendered_result is not None and not p.rendered_result.isspace() + ] + + if len(indexable_paragraph) == 0: + return [] + + enc = tiktoken.get_encoding("cl100k_base") + total_token = sum([len(enc.encode(c["content"])) for c in indexable_paragraph]) + + if total_token >= 8191: + logger.warning(f"Memory {memory.memory_id} has too many tokens {total_token}") + return [] + + content = [p["content"] for p in indexable_paragraph] + result = openai_client.embeddings.create(input = content, model = "text-embedding-3-small") + + neurons = [ + Neuron( + embedding = data.embedding, + content = indexable_paragraph[data.index]["content"], + memory_id = memory.memory_id, + position = Position(paragraph = indexable_paragraph[data.index]["idx"]), + embed_model = EmbedModel.OPENAI_TEXT_EMBEDDING_3_SMALL + ) + for data in result.data + ] + + return neurons + + +class NeuronBizSerivces: + def search_neurons(self, query: str, topk: int) -> List[NeuronDTO]: + if query is None or query == '': + return [] + + neurons = NeuronService().query_similar(query, topk) + + return NeuronDTO.from_model_list(neurons) + + def search_neurons_as_text(self, query, topk: int) -> str: + + if query is None or query == '': + return "" + + neurons = NeuronService().query_similar(query, topk) + + return "---\n".join([str(neuron) for neuron in neurons]) \ No newline at end of file diff --git a/packages/pkg-server/rag/migrations/0001_initial.py b/packages/pkg-server/rag/migrations/0001_initial.py index afff46a..52fc6d8 100644 --- a/packages/pkg-server/rag/migrations/0001_initial.py +++ b/packages/pkg-server/rag/migrations/0001_initial.py @@ -1,66 +1,127 @@ -# Generated by Django 5.0.6 on 2024-05-21 13:29 +# Generated by Django 5.0.6 on 2024-06-06 08:56 import pgvector.django import rag.domain.models from django.db import migrations, models - +from pgvector.django import VectorExtension class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ + VectorExtension(), + migrations.CreateModel( + name="Memory", + fields=[ + ("memory_id", models.AutoField(primary_key=True, serialize=False)), + ( + "memory_type", + models.CharField( + choices=[ + ("NOTE", "NOTE"), + ("REMINDER", "REMINDER"), + ("PIC", "PIC"), + ] + ), + ), + ("data", rag.domain.models.AppleNoteField(null=True)), + ("biz_id", models.CharField(max_length=100)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "memories", + }, + ), migrations.CreateModel( - name='Memory', + name="MemorySyncLog", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('memory_type', models.CharField(choices=[('NOTE', 'NOTE'), ('REMINDER', 'REMINDER'), ('PIC', 'PIC')])), - ('data', rag.domain.models.AppleNoteField(null=True)), - ('biz_id', models.CharField(max_length=100, null=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), + ("log_id", models.AutoField(primary_key=True, serialize=False)), + ("biz_id", models.CharField(max_length=100)), + ("biz_modified_at", models.DateTimeField(auto_now=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), ], options={ - 'db_table': 'memories', + "db_table": "memories_sync_log", + "ordering": ["-biz_modified_at"], }, ), migrations.CreateModel( - name='Neuron', + name="Neuron", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('content', models.TextField()), - ('embedding', pgvector.django.VectorField(dimensions=1536)), - ('memory_biz_id', models.CharField(max_length=100, null=True)), - ('position_in_memory', rag.domain.models.PositionField(null=True)), + ("neuron_id", models.AutoField(primary_key=True, serialize=False)), + ("content", models.TextField()), + ("embedding", pgvector.django.VectorField(dimensions=1536)), + ("memory_id", models.IntegerField()), + ("position", rag.domain.models.PositionField(null=True)), + ( + "embed_model", + models.CharField( + choices=[ + ( + "OPENAI_TEXT_EMBEDDING_3_LARGE", + "openai text-embedding-3-large", + ), + ( + "OPENAI_TEXT_EMBEDDING_3_SMALL", + "openai text-embedding-3-small", + ), + ] + ), + ), ], options={ - 'db_table': 'neurons', + "db_table": "neurons", }, ), migrations.CreateModel( - name='MemorySyncLog', + name="NeuronIndexLog", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('biz_id', models.CharField(max_length=100)), - ('biz_modified_at', models.DateTimeField(auto_now=True)), - ('updated_at', models.DateTimeField(auto_now=True)), - ('created_at', models.DateTimeField(auto_now_add=True)), + ("log_id", models.AutoField(primary_key=True, serialize=False)), + ("memory_id", models.IntegerField()), + ( + "state", + models.CharField( + choices=[ + ("NOT_STARTED", "not started"), + ("PROCESSING", "processing"), + ("INDEXED", "indexed"), + ], + default="NOT_STARTED", + ), + ), + ("indexed_at", models.DateTimeField(null=True)), + ("updated_at", models.DateTimeField(auto_now=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), ], options={ - 'db_table': 'memories_sync_log', - 'ordering': ['-biz_modified_at'], - 'indexes': [models.Index(fields=['biz_modified_at'], name='idx_biz_modified_at')], + "db_table": "neuron_index_logs", }, ), migrations.AddConstraint( - model_name='memorysynclog', - constraint=models.UniqueConstraint(fields=('biz_id',), name='uk_biz_id'), + model_name="memory", + constraint=models.UniqueConstraint(fields=("biz_id",), name="uk_biz_id"), + ), + migrations.AddIndex( + model_name="memorysynclog", + index=models.Index(fields=["biz_modified_at"], name="idx_biz_modified_at"), + ), + migrations.AddIndex( + model_name="neuron", + index=pgvector.django.HnswIndex( + ef_construction=64, + fields=["embedding"], + m=16, + name="idx_hnsw_embedding", + opclasses=["vector_cosine_ops"], + ), ), migrations.AddIndex( - model_name='neuron', - index=pgvector.django.HnswIndex(ef_construction=64, fields=['embedding'], m=16, name='idx_hnsw_embedding', opclasses=['vector_cosine_ops']), + model_name="neuronindexlog", + index=models.Index(fields=["memory_id"], name="idx_memory_id"), ), ] diff --git a/packages/pkg-server/rag/routers.py b/packages/pkg-server/rag/routers.py new file mode 100644 index 0000000..fae3135 --- /dev/null +++ b/packages/pkg-server/rag/routers.py @@ -0,0 +1,8 @@ +from ninja import Router + +from .views import memories_router, neurons_router + +router = Router() + +router.add_router("/memories", memories_router) +router.add_router("/neurons", neurons_router) \ No newline at end of file diff --git a/packages/pkg-server/rag/tests.py b/packages/pkg-server/rag/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/packages/pkg-server/rag/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/packages/pkg-server/rag/urls.py b/packages/pkg-server/rag/urls.py deleted file mode 100644 index f6de76d..0000000 --- a/packages/pkg-server/rag/urls.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.urls import path - -from .views import memory, neuron - -urlpatterns = [ - path("memories/list", memory.list_memories, name="list_memories"), - path("memories/sync", memory.sync_memories, name="sync_memories"), - path("neurons/search", neuron.search_neurons, name="search neurons"), -] \ No newline at end of file diff --git a/packages/pkg-server/rag/views/__init__.py b/packages/pkg-server/rag/views/__init__.py index e69de29..58e266f 100644 --- a/packages/pkg-server/rag/views/__init__.py +++ b/packages/pkg-server/rag/views/__init__.py @@ -0,0 +1,4 @@ +from .memory import router as memories_router +from .neuron import router as neurons_router + +__ALL__ = ["memories_router", "neurons_router"] \ No newline at end of file diff --git a/packages/pkg-server/rag/views/memory.py b/packages/pkg-server/rag/views/memory.py index ca42b64..6ac8ea6 100644 --- a/packages/pkg-server/rag/views/memory.py +++ b/packages/pkg-server/rag/views/memory.py @@ -1,108 +1,23 @@ -import itertools -from datetime import datetime from typing import List -from loci.domain.models.note import NoteContentParagraph -import openai -from rag.domain.enum import MemoryType -import tiktoken -from django.http import JsonResponse -from rest_framework import serializers +from ninja import Router -from loci.domain import Note -from loci.infra.fetchers.apple import AppleNotesFetcher -from loci.infra.renderers.markdown import MarkDown -from ..domain.models import Memory, MemorySyncLog, Neuron, Position -from ..dto.memory import MemoryDTO - -client = openai.OpenAI() - - -class MemorySerializer(serializers.ModelSerializer): - class Meta: - model = Memory - fields = '__all__' - -def list_memories(request): - - memories = Memory.objects.all() - - dto = MemoryDTO.from_model_list(memories) - - return JsonResponse({'data': dto, 'message': None, 'code': 200, 'success': True}, safe = False) - - -def sync_memories(request): - - markdown = MarkDown() - fetcher = AppleNotesFetcher(markdown) - fetcher.start() - - notes = fetcher.notes[0:10] - uuids = list(map(lambda x: x.uuid, notes)) - logs = MemorySyncLog.objects.filter(biz_id__in = uuids) - - last_modified_map_by_biz_id = {log.biz_id: log.biz_modified_at for log in logs} - - callback_min = datetime(1970, 1, 1).astimezone() - to_update_notes = list(filter( - lambda x: x.modified_at > last_modified_map_by_biz_id.get(x.uuid, callback_min), - notes - )) - - if len(to_update_notes) == 0: - return JsonResponse({'data': None, 'message': None, 'code': 200, 'success': True}) - - memories = [ - Memory(memory_type = MemoryType.NOTE, data = note.model_dump(mode = 'json'), biz_id = note.uuid) - for note in to_update_notes - ] - - neurons_mapper = map(generate_neurons, to_update_notes) - nonnull_neurons_mapper = [ - mapper - for mapper in neurons_mapper - if mapper is not None - ] - neurons = list(itertools.chain.from_iterable(nonnull_neurons_mapper)) - - sync_logs = [ - MemorySyncLog( - biz_id = note.uuid, - biz_modified_at = note.modified_at - ) - for note in to_update_notes - ] - - Memory.objects.bulk_create(memories) - MemorySyncLog.objects.bulk_create(sync_logs) - Neuron.objects.bulk_create(neurons) - return JsonResponse({'data': None, 'message': None, 'code': 200, 'success': True}) - - -def generate_neurons(note: Note) -> List[Neuron]: - - paragraph_list = note.content.paragraph_list - if len(paragraph_list) == 0: - return [] - # check if represent is not blank - indexable_paragraph = [{"idx":idx, "content": p.represent} for idx, p in enumerate(paragraph_list) if p.represent is not None and not p.represent.isspace()] - - if len(indexable_paragraph) == 0: - return [] +from ..dto.common import Result +from ..dto.memory import MemoryDTO +from ..infra.services import MemoryBizService - enc = tiktoken.get_encoding("cl100k_base") - total_token = sum([len(enc.encode(c["content"])) for c in indexable_paragraph]) +router = Router() - assert total_token <= 8191, "Too many tokens" +memory_biz_service = MemoryBizService() - content = [p["content"] for p in indexable_paragraph] - result = client.embeddings.create(input = content, model = "text-embedding-3-small") +@router.get('/list.json', response=Result[List[MemoryDTO]]) +def list_memories(request) -> Result[List[MemoryDTO]]: + dto = memory_biz_service.list_all() + return Result.with_data(dto) - neurons = [ - Neuron(embedding = data.embedding, content = indexable_paragraph[data.index]["content"], biz_id=note.uuid, position=Position(paragraph=indexable_paragraph[data.index]["idx"])) - for data in result.data - ] - return neurons \ No newline at end of file +@router.post('/sync.json', response=Result[None]) +def sync_memories(request) -> Result[None]: + memory_biz_service.sync_memories() + return Result.succ() \ No newline at end of file diff --git a/packages/pkg-server/rag/views/neuron.py b/packages/pkg-server/rag/views/neuron.py index 77c786f..522fb7d 100644 --- a/packages/pkg-server/rag/views/neuron.py +++ b/packages/pkg-server/rag/views/neuron.py @@ -1,32 +1,32 @@ +from typing import List + import openai -from django.http import JsonResponse -from pgvector.django import CosineDistance -from rest_framework import serializers -from rest_framework.decorators import api_view +from ninja import Router -from ..domain.models import Neuron +from ..domain.manager import NeuronManager +from ..dto.common import Result from ..dto.memory import NeuronDTO +from ..infra.services import NeuronBizSerivces client = openai.OpenAI() +router = Router() -class NeuronSerializer(serializers.ModelSerializer): - class Meta: - model = Neuron - fields = ['content'] +neuron_biz_services = NeuronBizSerivces() -@api_view(['GET']) -def search_neurons(request): +@router.post('/search.json', response=Result[List[NeuronDTO]]) +def search_neurons_json(request, query: str, topk: int) -> Result[List[NeuronDTO]]: + dto = neuron_biz_services.search_neurons(query=query, topk=topk) + return Result.with_data(dto) - query = request.query_params.get('q', None) - if query is None or query == '': - return JsonResponse({'data': None, 'message': 'q is required', 'code': 400, 'success': False}) - result = client.embeddings.create(input = query, model = "text-embedding-3-small") - embedding = result.data[0].embedding +@router.post('/search.text', response=Result[str]) +def search_neurons_text(request, query: str, topk: int) -> Result[str]: + result_str = neuron_biz_services.search_neurons_as_text(query=query, topk=topk) + return Result.with_data(result_str) - neurons = Neuron.objects.alias(distance = CosineDistance('embedding', embedding)).filter(distance__lt = 0.80) - dto = NeuronDTO.from_model_list(neurons) - return JsonResponse({'data': dto, 'message': None, 'code': 200, 'success': True}) +@router.post('/summrize.text', response=Result[str]) +def summrize_neurons(request, query: str) -> Result[str]: + return Result.with_data("") diff --git a/packages/pkg-server/tests/__init__.py b/packages/pkg-server/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/pkg-server/tests/test_servie_neuron.py b/packages/pkg-server/tests/test_servie_neuron.py new file mode 100644 index 0000000..7796f5a --- /dev/null +++ b/packages/pkg-server/tests/test_servie_neuron.py @@ -0,0 +1,16 @@ +import unittest +from rag.domain.models import Neuron +from rag.infra.services import NeuronService + + +class NeruonServiceTests(unittest.TestCase): + + def test_neuron_rerank(self): + + neurons = [ + Neuron(content = "test1"), + Neuron(content = "test2"), + Neuron(content = "great"), + ] + result = NeuronService.rerank_neurons(neurons = neurons, query = "test", topk = 2) + assert len(result) == 2 \ No newline at end of file diff --git a/pdm.lock b/pdm.lock index 2eebd19..0aff110 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:213253cda445d2a671b0f0bec033724025b56e35a57c30b6f921c5a7f10baacf" +content_hash = "sha256:45c18482da8cd2f81a2bb5fa85f796127f060708ddb1b4ee0c6b7b36b10f1256" [[package]] name = "agent" @@ -17,8 +17,8 @@ summary = "Default template for PDM package" groups = ["dev"] dependencies = [ "OpenAI>=1.30.1", + "django-ninja>=1.1.0", "django>=5.0.6", - "djangorestframework>=3.15.1", "pgvector>=0.2.5", "psycopg[binary]>=3.1.19", "python-dotenv>=1.0.1", @@ -122,7 +122,7 @@ summary = "Default template for PDM package" groups = ["dev"] dependencies = [ "click>=8.1.7", - "loci", + "koma", "tomlkit>=0.12.4", ] @@ -190,27 +190,42 @@ files = [ {file = "Django-5.0.6.tar.gz", hash = "sha256:ff1b61005004e476e0aeea47c7f79b85864c70124030e95146315396f1e7951f"}, ] +[[package]] +name = "django-ninja" +version = "1.1.0" +requires_python = ">=3.7" +summary = "Django Ninja - Fast Django REST framework" +groups = ["dev"] +dependencies = [ + "Django>=3.1", + "pydantic<3.0.0,>=2.0", +] +files = [ + {file = "django_ninja-1.1.0-py3-none-any.whl", hash = "sha256:6330c3497061d9fd1f43c1200f85c13aab7687110e2899f8304e5aa476c10b44"}, + {file = "django_ninja-1.1.0.tar.gz", hash = "sha256:87bff046416a2653ed2fbef1408e101292bf8170684821bac82accfd73bef059"}, +] + [[package]] name = "django-stubs" -version = "5.0.0" +version = "5.0.2" requires_python = ">=3.8" summary = "Mypy stubs for Django" groups = ["dev"] dependencies = [ "asgiref", "django", - "django-stubs-ext>=5.0.0", + "django-stubs-ext>=5.0.2", "types-PyYAML", - "typing-extensions", + "typing-extensions>=4.11.0", ] files = [ - {file = "django_stubs-5.0.0-py3-none-any.whl", hash = "sha256:084484cbe16a6d388e80ec687e46f529d67a232f3befaf55c936b3b476be289d"}, - {file = "django_stubs-5.0.0.tar.gz", hash = "sha256:b8a792bee526d6cab31e197cb414ee7fa218abd931a50948c66a80b3a2548621"}, + {file = "django_stubs-5.0.2-py3-none-any.whl", hash = "sha256:cb0c506cb5c54c64612e4a2ee8d6b913c6178560ec168009fe847c09747c304b"}, + {file = "django_stubs-5.0.2.tar.gz", hash = "sha256:236bc5606e5607cb968f92b648471f9edaa461a774bc013bf9e6bff8730f6bdf"}, ] [[package]] name = "django-stubs-ext" -version = "5.0.0" +version = "5.0.2" requires_python = ">=3.8" summary = "Monkey-patching and extensions for django-stubs" groups = ["dev"] @@ -219,38 +234,24 @@ dependencies = [ "typing-extensions", ] files = [ - {file = "django_stubs_ext-5.0.0-py3-none-any.whl", hash = "sha256:8e1334fdf0c8bff87e25d593b33d4247487338aaed943037826244ff788b56a8"}, - {file = "django_stubs_ext-5.0.0.tar.gz", hash = "sha256:5bacfbb498a206d5938454222b843d81da79ea8b6fcd1a59003f529e775bc115"}, + {file = "django_stubs_ext-5.0.2-py3-none-any.whl", hash = "sha256:8d8efec5a86241266bec94a528fe21258ad90d78c67307f3ae5f36e81de97f12"}, + {file = "django_stubs_ext-5.0.2.tar.gz", hash = "sha256:409c62585d7f996cef5c760e6e27ea3ff29f961c943747e67519c837422cad32"}, ] [[package]] name = "django-stubs" -version = "5.0.0" +version = "5.0.2" extras = ["compatible-mypy"] requires_python = ">=3.8" summary = "Mypy stubs for Django" groups = ["dev"] dependencies = [ - "django-stubs==5.0.0", + "django-stubs==5.0.2", "mypy~=1.10.0", ] files = [ - {file = "django_stubs-5.0.0-py3-none-any.whl", hash = "sha256:084484cbe16a6d388e80ec687e46f529d67a232f3befaf55c936b3b476be289d"}, - {file = "django_stubs-5.0.0.tar.gz", hash = "sha256:b8a792bee526d6cab31e197cb414ee7fa218abd931a50948c66a80b3a2548621"}, -] - -[[package]] -name = "djangorestframework" -version = "3.15.1" -requires_python = ">=3.6" -summary = "Web APIs for Django, made easy." -groups = ["dev"] -dependencies = [ - "django>=3.0", -] -files = [ - {file = "djangorestframework-3.15.1-py3-none-any.whl", hash = "sha256:3ccc0475bce968608cf30d07fb17d8e52d1d7fc8bfe779c905463200750cbca6"}, - {file = "djangorestframework-3.15.1.tar.gz", hash = "sha256:f88fad74183dfc7144b2756d0d2ac716ea5b4c7c9840995ac3bfd8ec034333c1"}, + {file = "django_stubs-5.0.2-py3-none-any.whl", hash = "sha256:cb0c506cb5c54c64612e4a2ee8d6b913c6178560ec168009fe847c09747c304b"}, + {file = "django_stubs-5.0.2.tar.gz", hash = "sha256:236bc5606e5607cb968f92b648471f9edaa461a774bc013bf9e6bff8730f6bdf"}, ] [[package]] @@ -338,9 +339,20 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +requires_python = ">=3.7" +summary = "brain-dead simple config-ini parsing" +groups = ["dev"] +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "ipython" -version = "8.24.0" +version = "8.25.0" requires_python = ">=3.10" summary = "IPython: Productive Interactive Computing" groups = ["dev"] @@ -356,8 +368,8 @@ dependencies = [ "traitlets>=5.13.0", ] files = [ - {file = "ipython-8.24.0-py3-none-any.whl", hash = "sha256:d7bf2f6c4314984e3e02393213bab8703cf163ede39672ce5918c51fe253a2a3"}, - {file = "ipython-8.24.0.tar.gz", hash = "sha256:010db3f8a728a578bb641fdd06c063b9fb8e96a9464c63aec6310fbcb5e80501"}, + {file = "ipython-8.25.0-py3-none-any.whl", hash = "sha256:53eee7ad44df903a06655871cbab66d156a051fd86f3ec6750470ac9604ac1ab"}, + {file = "ipython-8.25.0.tar.gz", hash = "sha256:c6ed726a140b6e725b911528f80439c534fac915246af3efc39440a6b0f9d716"}, ] [[package]] @@ -374,20 +386,6 @@ files = [ {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, ] -[[package]] -name = "loci" -version = "0.1.0" -requires_python = ">=3.12" -editable = true -path = "./packages/pkg-core" -summary = "Default template for PDM package" -groups = ["dev"] -dependencies = [ - "protobuf>=5.26.1", - "psycopg[binary]>=3.1.19", - "sqlalchemy[asyncio]>=2.0.30", -] - [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -433,6 +431,20 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "koma" +version = "0.1.0" +requires_python = ">=3.12" +editable = true +path = "./packages/pkg-core" +summary = "Default template for PDM package" +groups = ["dev"] +dependencies = [ + "protobuf>=5.26.1", + "psycopg[binary]>=3.1.19", + "sqlalchemy[asyncio]>=2.0.30", +] + [[package]] name = "numpy" version = "1.26.4" @@ -474,6 +486,17 @@ files = [ {file = "openai-1.30.1.tar.gz", hash = "sha256:4f85190e577cba0b066e1950b8eb9b11d25bc7ebcc43a86b326ce1bfa564ec74"}, ] +[[package]] +name = "packaging" +version = "24.0" +requires_python = ">=3.7" +summary = "Core utilities for Python packages" +groups = ["dev"] +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + [[package]] name = "parso" version = "0.8.4" @@ -512,6 +535,17 @@ files = [ {file = "pgvector-0.2.5-py2.py3-none-any.whl", hash = "sha256:5e5e93ec4d3c45ab1fa388729d56c602f6966296e19deee8878928c6d567e41b"}, ] +[[package]] +name = "pluggy" +version = "1.5.0" +requires_python = ">=3.8" +summary = "plugin and hook calling mechanisms for python" +groups = ["dev"] +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.43" @@ -684,6 +718,23 @@ files = [ {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, ] +[[package]] +name = "pytest" +version = "8.2.2" +requires_python = ">=3.8" +summary = "pytest: simple powerful testing with Python" +groups = ["dev"] +dependencies = [ + "colorama; sys_platform == \"win32\"", + "iniconfig", + "packaging", + "pluggy<2.0,>=1.5", +] +files = [ + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, +] + [[package]] name = "python-dotenv" version = "1.0.1" @@ -739,28 +790,28 @@ files = [ [[package]] name = "ruff" -version = "0.4.4" +version = "0.4.8" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." groups = ["dev"] files = [ - {file = "ruff-0.4.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:29d44ef5bb6a08e235c8249294fa8d431adc1426bfda99ed493119e6f9ea1bf6"}, - {file = "ruff-0.4.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c4efe62b5bbb24178c950732ddd40712b878a9b96b1d02b0ff0b08a090cbd891"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c8e2f1e8fc12d07ab521a9005d68a969e167b589cbcaee354cb61e9d9de9c15"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:60ed88b636a463214905c002fa3eaab19795679ed55529f91e488db3fe8976ab"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b90fc5e170fc71c712cc4d9ab0e24ea505c6a9e4ebf346787a67e691dfb72e85"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:8e7e6ebc10ef16dcdc77fd5557ee60647512b400e4a60bdc4849468f076f6eef"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9ddb2c494fb79fc208cd15ffe08f32b7682519e067413dbaf5f4b01a6087bcd"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c51c928a14f9f0a871082603e25a1588059b7e08a920f2f9fa7157b5bf08cfe9"}, - {file = "ruff-0.4.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5eb0a4bfd6400b7d07c09a7725e1a98c3b838be557fee229ac0f84d9aa49c36"}, - {file = "ruff-0.4.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b1867ee9bf3acc21778dcb293db504692eda5f7a11a6e6cc40890182a9f9e595"}, - {file = "ruff-0.4.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1aecced1269481ef2894cc495647392a34b0bf3e28ff53ed95a385b13aa45768"}, - {file = "ruff-0.4.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9da73eb616b3241a307b837f32756dc20a0b07e2bcb694fec73699c93d04a69e"}, - {file = "ruff-0.4.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:958b4ea5589706a81065e2a776237de2ecc3e763342e5cc8e02a4a4d8a5e6f95"}, - {file = "ruff-0.4.4-py3-none-win32.whl", hash = "sha256:cb53473849f011bca6e754f2cdf47cafc9c4f4ff4570003a0dad0b9b6890e876"}, - {file = "ruff-0.4.4-py3-none-win_amd64.whl", hash = "sha256:424e5b72597482543b684c11def82669cc6b395aa8cc69acc1858b5ef3e5daae"}, - {file = "ruff-0.4.4-py3-none-win_arm64.whl", hash = "sha256:39df0537b47d3b597293edbb95baf54ff5b49589eb7ff41926d8243caa995ea6"}, - {file = "ruff-0.4.4.tar.gz", hash = "sha256:f87ea42d5cdebdc6a69761a9d0bc83ae9b3b30d0ad78952005ba6568d6c022af"}, + {file = "ruff-0.4.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066"}, + {file = "ruff-0.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa"}, + {file = "ruff-0.4.8-py3-none-win32.whl", hash = "sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9"}, + {file = "ruff-0.4.8-py3-none-win_amd64.whl", hash = "sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d"}, + {file = "ruff-0.4.8-py3-none-win_arm64.whl", hash = "sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780"}, + {file = "ruff-0.4.8.tar.gz", hash = "sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268"}, ] [[package]] @@ -973,7 +1024,7 @@ files = [ [[package]] name = "uvicorn" -version = "0.29.0" +version = "0.30.1" requires_python = ">=3.8" summary = "The lightning-fast ASGI server." groups = ["dev"] @@ -982,8 +1033,8 @@ dependencies = [ "h11>=0.8", ] files = [ - {file = "uvicorn-0.29.0-py3-none-any.whl", hash = "sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de"}, - {file = "uvicorn-0.29.0.tar.gz", hash = "sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0"}, + {file = "uvicorn-0.30.1-py3-none-any.whl", hash = "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81"}, + {file = "uvicorn-0.30.1.tar.gz", hash = "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 44b2dbd..e8cbca5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,6 @@ [project] +name = "koma-project" +version = "0.1.0" requires-python = ">=3.12" [tool.pdm.dev-dependencies] @@ -12,16 +14,20 @@ dev = [ "uvicorn>=0.29.0", "django-stubs[compatible-mypy]>=5.0.0", "ipython>=8.24.0", + "pytest>=8.2.1", ] [tool.pdm.scripts] ruff = "ruff check" -mypy = "mypy packages" -lint = {composite = ["ruff", "mypy"]} +mypy_core = "mypy packages/pkg-core" +mypy_api = "mypy packages/pkg-server" +mypy = {composite = ["mypy_core", "mypy_api"]} +lint = {composite = ["mypy", "ruff"]} -server_test.cmd = "uvicorn agent.asgi:application" -server_test.env_file = ".env.test" +unit_test = "pytest packages/pkg-core/tests" + +server_test.cmd = "uvicorn agent.asgi:application --host 0.0.0.0 --reload --env-file ./.env.test" django_manage.cmd = "python packages/pkg-server/manage.py {args}" django_manage.env_file = ".env.test" @@ -41,3 +47,8 @@ django_settings_module = "agent.settings" [tool.ruff.lint] fixable = ["ALL"] ignore = ["F401"] + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore:.*custom tp_new.*in Python 3.14.*:DeprecationWarning", +] \ No newline at end of file diff --git a/tests/EntityTests.py b/tests/EntityTests.py deleted file mode 100644 index a7a432e..0000000 --- a/tests/EntityTests.py +++ /dev/null @@ -1,16 +0,0 @@ -import datetime -import unittest -from unittest import IsolatedAsyncioTestCase - -from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker -from sqlalchemy.orm import Session - -import tracemalloc - -tracemalloc.start() - - -class TestEntity(IsolatedAsyncioTestCase): - - async def test_log(self): - pass diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 24fae26..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -import unittest - - -if __name__ == '__main__': - unittest.main()