fix: make proWES run locally

.github/workflows/main.yml

@@ -45,8 +45,14 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
+      - name: Install yq
+        run: pip install yq
+      - name: Update configuration hosts
+        run: |
+          yq -y --in-place '.db.host = "mongodb"' pro_wes/config.yaml
+          yq -y --in-place '.jobs.host = "rabbitmq"' pro_wes/config.yaml
       - name: Deploy app
-        run: docker-compose up -d --build
+        run: docker compose up -d --build
       - name: Wait for app startup
         run: sleep 20
       - name: Probe endpoint

.gitignore

@@ -221,3 +221,4 @@ tags
 !.gitkeep
 celeryd.pid
 *.modified.yaml
+data

Makefile

@@ -0,0 +1,69 @@
+default: help
+
+# Color variables
+BOLD = \033[1m
+PURPLE = \033[35m
+GRAY = \033[37m
+CYAN = \033[36m
+NC = \033[0m
+
+.PHONY: help
+help:
+	@echo "\nUsage: make [target] ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"
+	@echo "Available targets:\n"
+	@echo "Development Environment ---------------------------------------------------"
+	@echo "  ${BOLD}${PURPLE}clean-venv${NC} ${GRAY}(cv)${NC}: ${CYAN}Remove virtual environment.${NC}"
+	@echo "  ${BOLD}${PURPLE}install${NC} ${GRAY}(i)${NC}: ${CYAN}Install package in development mode.${NC}"
+	@echo "  ${BOLD}${PURPLE}venv${NC} ${GRAY}(v)${NC}: ${CYAN}Create virtual environment.${NC}\n"
+	@echo "Development Services -----------------------------------------------------"
+	@echo "  ${BOLD}${PURPLE}dev-app${NC} ${GRAY}(da)${NC}: ${CYAN}Run only the prowes Flask app locally.${NC}"
+	@echo "  ${BOLD}${PURPLE}dev-celery${NC} ${GRAY}(dc)${NC}: ${CYAN}Run only the Celery worker locally.${NC}"
+	@echo "  ${BOLD}${PURPLE}dev-docker${NC} ${GRAY}(dd)${NC}: ${CYAN}Run all docker dev services.${NC}"
+
+# Development Environment
+.PHONY: clean-venv cv
+clean-venv cv:
+	@echo "Removing virtual environment..."
+	rm -rf .venv
+
+.PHONY: install i
+install i:
+	@echo "Installing package in development mode..."
+	pip install -e .
+
+.PHONY: venv v
+venv v:
+	@echo "Creating virtual environment..."
+	python -m venv .venv
+	@echo "Run 'source .venv/bin/activate' to activate the virtual environment"
+
+# Development Services
+.PHONY: dev-docker dd
+dev-docker dd:
+	@echo "Starting Flask app..."
+	docker compose -f docker-compose.dev.yaml up -d
+
+.PHONY: dev-app da
+dev-app da:
+	@echo "Starting FOCA app..."
+	python pro_wes/app.py
+
+.PHONY: dev-celery dc
+dev-celery dc:
+	@echo "Starting Celery worker..."
+	cd pro_wes && celery -A celery_worker worker -E --loglevel=info
+
+.PHONY: lint fl
+lint fl:
+
+	@echo "1.Formatting with black..."
+	black --exclude .venv pro_wes/ setup.py tests/
+	@echo "\n\n2.Checking style with flake8..."
+	black --exclude .venv pro_wes/ setup.py tests/
+	@echo "\n\n3.Running pylint..."
+	pylint pro_wes/ setup.py
+
+.PHONY: type-check tc
+type-check tc:
+	@echo "Running mypy..."
+	mypy pro_wes/ setup.py

docker-compose.dev.yaml

@@ -0,0 +1,23 @@
+services:
+  rabbitmq:
+    image: rabbitmq:3-management
+    restart: unless-stopped
+    links:
+      - mongodb
+    ports:
+      - "5672:5672"
+
+  mongodb:
+    image: mongo:3.2
+    restart: unless-stopped
+    volumes:
+      - ${PROWES_DATA_DIR:-../data/pro_wes}/db:/data/db
+    ports:
+      - "27017:27017"
+
+  flower:
+    image: mher/flower:0.9.7
+    restart: unless-stopped
+    command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555
+    ports:
+      - "5555:5555"

docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3.6'
 services:
 
   prowes-worker:
@@ -22,7 +21,7 @@ services:
     restart: unless-stopped
     links:
       - mongodb
-    command: bash -c "cd /app/pro_wes; gunicorn -c gunicorn.py wsgi:app"
+    command: bash -c "cd /app; gunicorn -c pro_wes/gunicorn.py pro_wes.wsgi:app"
     ports:
       - "8090:8080"
     volumes:
@@ -34,15 +33,15 @@ services:
     links:
       - mongodb
     ports:
-      - "5682:5672"
+      - "5672:5672"
 
   mongodb:
     image: mongo:3.2
     restart: unless-stopped
     volumes:
       - ${PROWES_DATA_DIR:-../data/pro_wes}/db:/data/db
     ports:
-      - "27027:27017"
+      - "27017:27017"
 
   flower:
     image: mher/flower:0.9.7
@@ -51,4 +50,4 @@ services:
       - prowes-worker
     command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555
     ports:
-      - "5565:5555"
+      - "5555:5555"

pro_wes/config.yaml

@@ -30,7 +30,7 @@ security:
 # Database configuration
 # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.DBConfig
 db:
-  host: mongodb
+  host: localhost
   port: 27017
   dbs:
     runStore:
@@ -53,8 +53,8 @@ db:
 api:
   specs:
     - path:
-        - api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml
-        - api/additions.openapi.yaml
+        - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml
+        - pro_wes/api/additions.openapi.yaml
       add_operation_fields:
         x-openapi-router-controller: pro_wes.ga4gh.wes.controllers
       add_security_fields:
@@ -92,7 +92,7 @@ log:
 # Background job configuration
 # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.JobsConfig
 jobs:
-  host: rabbitmq
+  host: localhost
   port: 5672
   backend: "rpc://"
   include:
@@ -116,7 +116,7 @@ custom:
     id_length: 6
     polling_attempts: 100
     polling_wait: 3
-    storage_path: "/data"
+    storage_path: "./data"
     timeout_job: null
     timeout_post: null
   list_runs:

pro_wes/ga4gh/wes/workflow_runs.py

@@ -482,7 +482,9 @@ def _validate_run_request(
         """
         dict_of_lists = form_data.to_dict(flat=False)
         # flatten single item lists
-        dict_atomic = {k: v[0] if len(v) == 1 else v for k, v in dict_of_lists.items()}
+        dict_atomic: dict = {
+            k: v[0] if len(v) == 1 else v for k, v in dict_of_lists.items()
+        }
         # remove 'workflow_attachment' field
         dict_atomic.pop("workflow_attachment", None)
         model_instance = RunRequest(**dict_atomic)

requirements.txt

@@ -1,4 +1,4 @@
 connexion<3
 email-validator>=2.1.0,<3
-foca>=0.12.1
+foca==0.12.1
 gunicorn>=20.1.0,<21