Merge pull request #1 from AFutureD/feat-ninjia

Using ninjia framework

.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+  "image": "mcr.microsoft.com/devcontainers/python:3.12",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "GitHub.copilot",
+        "ms-python.mypy-type-checker",
+        "ms-python.vscode-pylance",
+        "ms-python.debugpy",
+        "charliermarsh.ruff",
+        "vscode-icons-team.vscode-icons"
+      ]
+    }
+  },
+  "forwardPorts": [8000],
+  "postCreateCommand": "pipx install pdm; pdm install;"
+}

.env

@@ -0,0 +1,10 @@
+SERVER_URL='https://example.com'
+
+PGHOST='*.neon.tech'
+PGDATABASE='db_name'
+PGUSER='user_name'
+PGPASSWORD='<password>'
+
+OPENAI_API_KEY='sk-xxx'
+
+CO_API_KEY='xxxxx'

.vscode/settings.json

@@ -4,8 +4,20 @@
     "files.exclude": {
       "**/__pycache__": true,
       "**/.mypy_cache": true,
-      ".pdm-python": true,
+      "**/.pdm-python": true,
       "**/.pdm-build": true,
-      ".venv": true 
-    }
+      "**/.venv": true,
+      "**/.idea": true,
+      "**/.ruff_cache": true,
+      "**/.pytest_cache": true,
+    },
+    "python.analysis.inlayHints.functionReturnTypes": true,
+    "python.analysis.inlayHints.variableTypes": true,
+    "python.analysis.inlayHints.pytestParameters": true,
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestArgs": [
+      "packages"
+    ],
+    "terminal.integrated.shellIntegration.history": 100000,
   }

README.md

@@ -1,35 +1,66 @@
-# loci
-
-> The method of loci is a strategy for memory enhancement, 
-> which uses visualizations of familiar spatial environments in order to enhance the recall of information. 
-> The method of loci is also known as the memory journey, 
-> memory palace, journey method, memory spaces, or mind palace technique.
-> 
-> -- wikipedia
- 
+# koma
+
 ## Overview
 
 Notice: This project is still in development.
 
-This project(`loci`) is designed to help users search for information on macOS.
+This project(`koma`) is designed to help users search for information on macOS.
 
-Specifically thanks to the [apple_cloud_notes_parser](https://github.com/threeplanetssoftware/apple_cloud_notes_parser).
+Specifically thanks to the [apple_cloud_notes_parser](https://github.com/threeplanetssoftware/apple_cloud_notes_parser) and [neon](https://neon.tech).
 
 
 ## Feature
 
-in progress:
+- [x] [core] List all notes from Notes.app;
+- [x] [core] Convert all notes into markdown;
+- [ ] [core] Get a specific note;
+- [ ] [core] Create a note using AppleScript;
+- [x] [api ] Incremental index all notes and its paragraphs;
+- [x] [api ] Semantic query in both json and plain text;
+- [ ] [api ] Summrize similar content;
+- [ ] [api ] Automatic sync Apple notes.
+
+## Usage 
+
+### As a data connector in RAG platform.
+
+By using OpenAPI this is easy to intergate into any rag platform, such as [Dify](https://dify.ai).
+
+In Dify, goto [Tools](https://cloud.dify.ai/tools?category=api) and create one using `<Your Domain>/api/openapi.json`.
+
+### Raycast
+
+TODO.
+
+## Install
+
+### Step 0: Prepare
+
+1. You have [pdm](https://pdm-project.org/en/stable/) and [uvicorn](https://www.uvicorn.org) installed.
+2. Your Terminal have [Full Disk Access](https://www.perplexity.ai/search/How-to-enable-mOAW4vpVRlmeMvtg6EjnNw) permission.
+3. Your Apple Notes.app folder is `~/Library/Group Containers/group.com.apple.notes`
+4. Configure the `.env`
+
+### Step 1: download this project
+
+```
+> git clone https://github.com/AFutureD/koma.git
+```
+
+### Step 2: Install dependencies.
+
+```
+> cd koma
+> pdm sync
+```
 
-- [x] [core] get notes from notes.app
-- [x] [core] render notes into markdown
-- [x] [core] incremental save notes into database(current mongodb only)
-- [x] [cli ] list notes
-- [ ] [cli ] get the content of a note from database
-- [ ] [cli ] search notes
+### Step 3: Run server
 
+```
+> pdm run django_manage migrate rag
+> uvicorn agent.asgi:application --host 0.0.0.0 --env-file ./.env
+```
 
-TODO:
 
-- [ ] [core] using pyiCloud. 
 
 

packages/pkg-cli/pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 dependencies = [
     "click>=8.1.7",
     "tomlkit>=0.12.4",
-    "loci"
+    "koma"
 ]
 requires-python = ">=3.12"
 readme = "README.md"

packages/pkg-core/koma/core/__init__.py

@@ -0,0 +1,6 @@
+from .render import RenderAble, TextRenderer
+from .model import Model
+
+__all__ = [
+    'Model', 'TextRenderer', 'RenderAble'
+]

packages/pkg-core/koma/core/model.py

@@ -0,0 +1,6 @@
+from typing import Any
+from pydantic import BaseModel, Field
+
+class Model(BaseModel):
+    metadata: dict[str, Any] = Field(default=dict(), exclude=True)
+

packages/pkg-core/loci/core/render.py → packages/pkg-core/koma/core/render.py

@@ -1,24 +1,27 @@
 from __future__ import annotations
+
 from abc import ABC, abstractmethod
+from pydantic import Field
 
-from pydantic import BaseModel
+from .model import Model
 
 
-class BaseRenderer(ABC):
+class TextRenderer(ABC):
+    META_KEY = "TEXT_RENDER_RESULT"
 
     @abstractmethod
     def render(self, renderable: RenderAble) -> str:
-        pass
+        ...
 
 
-class RenderAble(BaseModel):
-    rendered: bool = False
+class RenderAble(Model):
 
-    represent: str | None = None
+    rendered: bool = Field(default=False, exclude=True)
+    rendered_result: str | None = Field(default=None, exclude=True)
 
-    def render(self, renderer: BaseRenderer):
+    def render(self, renderer: TextRenderer):
         if self.rendered:
-            return self.represent
+            return
 
         for prop in self.__dict__.values():
             if isinstance(prop, RenderAble):
@@ -33,8 +36,9 @@ def render(self, renderer: BaseRenderer):
                         item.render(renderer)
 
         self.rendered = True
-        if isinstance(self, RenderAble):
-            represent = renderer.render(self)
-            self.represent = represent
-        else:
-            raise RuntimeError(f"Unknown type {type(self)}")
+        assert isinstance(self, RenderAble), f"{self.__class__.__qualname__} must be RenderAble"
+
+        render_result = renderer.render(self)
+
+        self.metadata[renderer.META_KEY] = render_result
+        self.rendered_result = render_result

packages/pkg-core/loci/domain/__init__.py → packages/pkg-core/koma/domain/__init__.py

@@ -1,6 +1,6 @@
 from .models.note import Note, NoteContent, NoteContentLine, NoteContentParagraph
 from .models.attachment import NoteAttachment, NoteAttachmentLink, NoteAttachmentMedia, NoteAttachmentTag, NoteAttachmentDraw, NoteAttachmentGallery, NoteAttachmentTable, NoteAttachmentTableCell
-from .models.text import ParagraphStyleType, FontStyle, CheckInfo, ParagraphStyle, TextAttribute, AttributeText
+from .models.style import ParagraphStyleType, FontStyle, CheckInfo, ParagraphStyle, TextAttribute, AttributeText
 
 __all__ = [
     "ParagraphStyleType", "FontStyle", "CheckInfo", "ParagraphStyle", "TextAttribute", "AttributeText",

packages/pkg-core/loci/domain/models/attachment.py → packages/pkg-core/koma/domain/models/attachment.py

@@ -1,15 +1,15 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import List, Dict
+from typing import List, Dict, Protocol
 
 from pydantic import BaseModel, ConfigDict
 from pydantic._internal._model_construction import ModelMetaclass
 
-from ...core import RenderAble
+from ...core import RenderAble, Model
 
 
-class NoteAttachmentTableCell(BaseModel):
+class NoteAttachmentTableCell(Model):
     column: int
     row: int
     text: str
@@ -23,19 +23,20 @@ def __repr__(self):
 
 class NoteAttachmentFactory(ModelMetaclass):
 
-    def __call__(cls, type_uti: str, *args, **kwargs):
+    def __call__(cls, *args, **kwargs):
+        type_uti = kwargs.get("type_uti")
         if type_uti == "public.url":
-            return NoteAttachmentLink(type_uti = type_uti, **kwargs)
+            return NoteAttachmentLink(**kwargs)
         elif type_uti in ["com.apple.paper", "com.apple.drawing.2", "com.apple.drawing"]:
-            return NoteAttachmentDraw(type_uti = type_uti, **kwargs)
+            return NoteAttachmentDraw(**kwargs)
         elif type_uti in ["com.apple.notes.table"]:
-            return NoteAttachmentTable(type_uti = type_uti, **kwargs)
+            return NoteAttachmentTable(**kwargs)
         elif type_uti in ["com.apple.notes.inlinetextattachment.hashtag"]:
-            return NoteAttachmentTag(type_uti = type_uti, **kwargs)
+            return NoteAttachmentTag(**kwargs)
         elif type_uti == "com.apple.notes.gallery":
-            return NoteAttachmentGallery(type_uti = type_uti, **kwargs)
+            return NoteAttachmentGallery(**kwargs)
         else:
-            return NoteAttachmentMedia(type_uti = type_uti, **kwargs)
+            return NoteAttachmentMedia(**kwargs)
 
 
 class NoteAttachmentMetaClass(NoteAttachmentFactory):
@@ -44,7 +45,7 @@ def __call__(cls, *args, **kwargs):
         return type.__call__(cls, *args, **kwargs)
 
 
-class NoteAttachment(RenderAble, BaseModel, metaclass=NoteAttachmentFactory):
+class NoteAttachment(RenderAble, Model, metaclass=NoteAttachmentFactory):
 
     type_uti: str
     z_pk: int
@@ -88,30 +89,28 @@ class NoteAttachmentMedia(NoteAttachment, metaclass=NoteAttachmentMetaClass):
     file_uuid: str
     file_name: str
 
-    def get_data_path(self) -> None | str:
-        if self.media_root_path is None:
-            return ""
-
-        root = Path(self.media_root_path)
+    def get_data_path(self) -> str:
+
         if self.generation is None:
-            data_path = root / "Media" / self.file_uuid / self.file_name
+            data_path = f"Media/{self.file_uuid}/{self.file_name}"
         else:
-            data_path = root / "Media" / self.file_uuid / self.generation / self.file_name
-        return str(data_path.absolute())
+            data_path = f"Media/{self.file_uuid}/{self.generation}/{self.file_name}"
+
+        return f"{self.media_root_path}/{data_path}"
 
 
 class NoteAttachmentDraw(NoteAttachment, metaclass=NoteAttachmentMetaClass):
     generation: None | str
     media_root_path: str
 
     def get_data_path(self) -> str:
-        root = Path(self.media_root_path)
 
         if self.generation is None:
-            data_path = root / "FallbackImages" / "{}.jpg".format(self.identifier)
+            data_path = f"FallbackImages/{self.identifier}.jpg"
         else:
-            data_path = root / "FallbackImages" / self.identifier / self.generation / "FallbackImage.png"
-        return str(data_path.absolute())
+            data_path = f"FallbackImages/{self.identifier}/{self.generation}/FallbackImage.png"
+
+        return f"{self.media_root_path}/{data_path}"
 
 
 class NoteAttachmentTable(NoteAttachment, metaclass=NoteAttachmentMetaClass):
@@ -123,5 +122,6 @@ class NoteAttachmentTable(NoteAttachment, metaclass=NoteAttachmentMetaClass):
 class NoteAttachmentTag(NoteAttachment, metaclass=NoteAttachmentMetaClass):
     pass
 
+
 class NoteAttachmentGallery(NoteAttachment, metaclass=NoteAttachmentMetaClass):
     pass