feat: clean up & refactor majorly

.dockerignore

@@ -1,5 +1,21 @@
-.coverage
-.pytest_cache/
-.tox/
+# Cache and dot files
+.coverage/
 .eggs/
-tesk-core/containers/
+.github/
+.gitignore
+.mypy_cache/
+.pytest_cache/
+.ruff/
+.ruff_cache/
+.venv/
+# Docs and artifacts
+./deployment
+./docs
+./images
+# Build
+./build
+./dist
+# Scripts and tools
+./scripts
+Makefile
+PULL_REQUEST_TEMPLATE.md

.github/ISSUE_TEMPLATE/general-purpose.md

@@ -0,0 +1,33 @@
+Your issue may already be reported!
+Please search on the [issue tracker](https://github.com/elixir-cloud-aai/TESK/issues) before creating one.
+
+## Expected Behavior
+<!--- If you're describing a bug, tell us what should happen -->
+<!--- If you're suggesting a change/improvement, tell us how it should work -->
+
+## Current Behavior
+<!--- If describing a bug, tell us what happens instead of the expected behavior -->
+<!--- If suggesting a change/improvement, explain the difference from current behavior -->
+
+## Possible Solution
+<!--- Not obligatory, but suggest a fix/reason for the bug, -->
+<!--- or ideas how to implement the addition or change -->
+
+## Steps to Reproduce (for bugs)
+<!--- Provide a link to a live example, or an unambiguous set of steps to -->
+<!--- reproduce this bug. Include code to reproduce, if relevant -->
+1.
+2.
+3.
+4.
+
+## Context
+<!--- How has this issue affected you? What are you trying to accomplish? -->
+<!--- Providing context helps us come up with a solution that is most useful in the real world -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->
+* Version used:
+* Browser Name and version:
+* Operating System and version (desktop or mobile):
+* Link to your project:

.github/workflows/build.yaml → .github/workflows/build_and_publish_charts.yaml

@@ -1,12 +1,18 @@
-name: Helm Chart Testing
+name: Build and Publish Helm Charts
 
 on:
-  pull_request: 
-    branches: [master]
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
-  helm:
+  create-cluster:
+    name: Create k3s Cluster
     runs-on: ubuntu-latest
+    needs: checkout
     steps:
       - name: Checkout
         uses: actions/[email protected]
@@ -20,31 +26,47 @@ jobs:
 
       - name: Create namespace
         run: kubectl create ns tesk
-
+
+  helm-deps:
+    name: Helm Dependencies
+    runs-on: ubuntu-latest
+    needs: checkout
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
       - name: Helm Deps
         run: |
-          for dir in $(ls -d charts/*); do
+          for dir in $(ls -d deployment/charts/*); do
             helm dependency update $dir;
           done
-
+
+  helm-lint:
+    name: Helm Lint
+    runs-on: ubuntu-latest
+    needs: [helm-deps]
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
       - name: Helm Lint
         run: |
-          for dir in $(ls -d charts/*); do
+          for dir in $(ls -d deployment/charts/*); do
             helm lint $dir
           done
-
-      - name: Apply Helm file
-        run: helm install -n tesk tesk . -f values.yaml
-        working-directory: charts/tesk
-
-      - name: Sleep for 30 seconds
-        run: sleep 30
-
-      - name: Get Helm and k8s
-        run: helm list -n tesk && kubectl get all -n tesk
-
-      - name: curl URL
-        run: curl localhost -vL
+
+  chart-releaser:
+    name: Run chart-releaser
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
 
       - name: Configure Git
         run: |
@@ -57,4 +79,3 @@ jobs:
           skip_existing: true
         env:
           CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-

.github/workflows/docker-build-publish-filer.yml → .github/workflows/build_and_publish_filer.yaml

@@ -1,4 +1,4 @@
-name: tesk-core-filer
+name: Build and Publish Filer
 
 on:
   push:
@@ -14,6 +14,7 @@ on:
 
 jobs:
   build-from-source:
+    name: Build and publish Docker image
     runs-on: ubuntu-latest
     steps:
       - name: Checkout elixir-cloud-aai/tesk-core
@@ -43,7 +44,7 @@ jobs:
         with:
           context: .
           push: true
-          file: ./containers/filer.Dockerfile
+          file: ./deployment/containers/filer.Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 

.github/workflows/docker-build-publish-taskmaster.yml → .github/workflows/build_and_publish_taskmaster.yaml

@@ -1,4 +1,4 @@
-name: tesk-core-taskmaster
+name: Build and Publish Taskmaster
 
 on:
   push:
@@ -43,7 +43,7 @@ jobs:
         with:
           context: .
           push: true
-          file: ./containers/taskmaster.Dockerfile
+          file: ./deployment/containers/taskmaster.Dockerfile
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 

.github/workflows/code_quality.yaml

@@ -0,0 +1,102 @@
+name: Code Quality
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        id: cq
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install lint dependencies
+        run: poetry install --only=lint --no-interaction --no-root
+
+      - name: Check code quality
+        run: poetry run ruff check .
+
+  format:
+    name: Format
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        id: cq
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install lint dependencies
+        run: poetry install --only=lint --no-interaction --no-root
+
+      - name: Check code style
+        run: poetry run ruff format --check
+
+  type-check:
+    name: Type Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up python
+        id: cqtc
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install types and mypy dependencies
+        run: poetry install --only=types --no-interaction --no-root
+
+      - name: Check types
+        run: poetry run mypy tesk/
+
+  spell-check:
+    name: Spell Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        id: cq
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install lint dependencies
+        run: poetry install --only=lint --no-interaction --no-root
+
+      - name: Check spellings
+        run: poetry run typos . 

.github/workflows/code_test_unit.yaml

@@ -0,0 +1,42 @@
+name: Code Test Unit
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  unit-test:
+    name: Unit Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        id: ct
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install test dependencies
+        run: poetry install --only=test --no-interaction
+
+      - name: Run tests and generate coverage as test_unit.xml
+        run: poetry run pytest --cov-report term --cov-report xml:test_unit.xml --cov=tests/test_unit
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: test_unit
+          files: ./test_unit.xml
+          fail_ci_if_error: true
+          verbose: true