diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b5d2c7..c80235a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,8 +45,14 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.10" + - name: Install yq + run: pip install yq + - name: Update configuration hosts + run: | + yq -y --in-place '.db.host = "mongodb"' pro_wes/config.yaml + yq -y --in-place '.jobs.host = "rabbitmq"' pro_wes/config.yaml - name: Deploy app - run: docker-compose up -d --build + run: docker compose up -d --build - name: Wait for app startup run: sleep 20 - name: Probe endpoint diff --git a/.gitignore b/.gitignore index be820fb..6c8395f 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,4 @@ tags !.gitkeep celeryd.pid *.modified.yaml +data \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..93e5a3c --- /dev/null +++ b/Makefile @@ -0,0 +1,69 @@ +default: help + +# Color variables +BOLD = \033[1m +PURPLE = \033[35m +GRAY = \033[37m +CYAN = \033[36m +NC = \033[0m + +.PHONY: help +help: + @echo "\nUsage: make [target] ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n" + @echo "Available targets:\n" + @echo "Development Environment ---------------------------------------------------" + @echo " ${BOLD}${PURPLE}clean-venv${NC} ${GRAY}(cv)${NC}: ${CYAN}Remove virtual environment.${NC}" + @echo " ${BOLD}${PURPLE}install${NC} ${GRAY}(i)${NC}: ${CYAN}Install package in development mode.${NC}" + @echo " ${BOLD}${PURPLE}venv${NC} ${GRAY}(v)${NC}: ${CYAN}Create virtual environment.${NC}\n" + @echo "Development Services -----------------------------------------------------" + @echo " ${BOLD}${PURPLE}dev-app${NC} ${GRAY}(da)${NC}: ${CYAN}Run only the prowes Flask app locally.${NC}" + @echo " ${BOLD}${PURPLE}dev-celery${NC} ${GRAY}(dc)${NC}: ${CYAN}Run only the Celery worker locally.${NC}" + @echo " ${BOLD}${PURPLE}dev-docker${NC} ${GRAY}(dd)${NC}: ${CYAN}Run all docker dev services.${NC}" + +# Development Environment +.PHONY: clean-venv cv +clean-venv cv: + @echo "Removing virtual environment..." + rm -rf .venv + +.PHONY: install i +install i: + @echo "Installing package in development mode..." + pip install -e . + +.PHONY: venv v +venv v: + @echo "Creating virtual environment..." + python -m venv .venv + @echo "Run 'source .venv/bin/activate' to activate the virtual environment" + +# Development Services +.PHONY: dev-docker dd +dev-docker dd: + @echo "Starting Flask app..." + docker compose -f docker-compose.dev.yaml up -d + +.PHONY: dev-app da +dev-app da: + @echo "Starting FOCA app..." + python pro_wes/app.py + +.PHONY: dev-celery dc +dev-celery dc: + @echo "Starting Celery worker..." + cd pro_wes && celery -A celery_worker worker -E --loglevel=info + +.PHONY: lint fl +lint fl: + + @echo "1.Formatting with black..." + black --exclude .venv pro_wes/ setup.py tests/ + @echo "\n\n2.Checking style with flake8..." + black --exclude .venv pro_wes/ setup.py tests/ + @echo "\n\n3.Running pylint..." + pylint pro_wes/ setup.py + +.PHONY: type-check tc +type-check tc: + @echo "Running mypy..." + mypy pro_wes/ setup.py \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml new file mode 100644 index 0000000..85111a9 --- /dev/null +++ b/docker-compose.dev.yaml @@ -0,0 +1,23 @@ +services: + rabbitmq: + image: rabbitmq:3-management + restart: unless-stopped + links: + - mongodb + ports: + - "5672:5672" + + mongodb: + image: mongo:3.2 + restart: unless-stopped + volumes: + - ${PROWES_DATA_DIR:-../data/pro_wes}/db:/data/db + ports: + - "27017:27017" + + flower: + image: mher/flower:0.9.7 + restart: unless-stopped + command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555 + ports: + - "5555:5555" diff --git a/docker-compose.yaml b/docker-compose.yaml index 01d896f..50c7dc0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '3.6' services: prowes-worker: @@ -22,7 +21,7 @@ services: restart: unless-stopped links: - mongodb - command: bash -c "cd /app/pro_wes; gunicorn -c gunicorn.py wsgi:app" + command: bash -c "cd /app; gunicorn -c pro_wes/gunicorn.py pro_wes.wsgi:app" ports: - "8090:8080" volumes: @@ -34,7 +33,7 @@ services: links: - mongodb ports: - - "5682:5672" + - "5672:5672" mongodb: image: mongo:3.2 @@ -42,7 +41,7 @@ services: volumes: - ${PROWES_DATA_DIR:-../data/pro_wes}/db:/data/db ports: - - "27027:27017" + - "27017:27017" flower: image: mher/flower:0.9.7 @@ -51,4 +50,4 @@ services: - prowes-worker command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555 ports: - - "5565:5555" + - "5555:5555" diff --git a/pro_wes/config.yaml b/pro_wes/config.yaml index 834376f..a0ff875 100644 --- a/pro_wes/config.yaml +++ b/pro_wes/config.yaml @@ -30,7 +30,7 @@ security: # Database configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.DBConfig db: - host: mongodb + host: localhost port: 27017 dbs: runStore: @@ -53,8 +53,8 @@ db: api: specs: - path: - - api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml - - api/additions.openapi.yaml + - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml + - pro_wes/api/additions.openapi.yaml add_operation_fields: x-openapi-router-controller: pro_wes.ga4gh.wes.controllers add_security_fields: @@ -92,7 +92,7 @@ log: # Background job configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.JobsConfig jobs: - host: rabbitmq + host: localhost port: 5672 backend: "rpc://" include: @@ -116,7 +116,7 @@ custom: id_length: 6 polling_attempts: 100 polling_wait: 3 - storage_path: "/data" + storage_path: "./data" timeout_job: null timeout_post: null list_runs: diff --git a/pro_wes/ga4gh/wes/workflow_runs.py b/pro_wes/ga4gh/wes/workflow_runs.py index d3fd27e..74b97a8 100644 --- a/pro_wes/ga4gh/wes/workflow_runs.py +++ b/pro_wes/ga4gh/wes/workflow_runs.py @@ -482,7 +482,9 @@ def _validate_run_request( """ dict_of_lists = form_data.to_dict(flat=False) # flatten single item lists - dict_atomic = {k: v[0] if len(v) == 1 else v for k, v in dict_of_lists.items()} + dict_atomic: dict = { + k: v[0] if len(v) == 1 else v for k, v in dict_of_lists.items() + } # remove 'workflow_attachment' field dict_atomic.pop("workflow_attachment", None) model_instance = RunRequest(**dict_atomic) diff --git a/requirements.txt b/requirements.txt index 2d6ddc5..14f3624 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ connexion<3 email-validator>=2.1.0,<3 -foca>=0.12.1 +foca==0.12.1 gunicorn>=20.1.0,<21