From 0b1548e5f7ca3886c3ce53b25be121382019469d Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 19 Dec 2024 00:42:44 +0530 Subject: [PATCH 1/6] WIP --- .gitignore | 1 + Makefile | 54 ++++++++++++++ docker-compose.dev.yaml | 23 ++++++ docker-compose.yaml | 3 +- pro_wes/app.py | 9 ++- pro_wes/config.dev.yaml | 154 ++++++++++++++++++++++++++++++++++++++++ pro_wes/config.yaml | 4 +- requirements.txt | 1 + 8 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 Makefile create mode 100644 docker-compose.dev.yaml create mode 100644 pro_wes/config.dev.yaml diff --git a/.gitignore b/.gitignore index be820fb..6c8395f 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,4 @@ tags !.gitkeep celeryd.pid *.modified.yaml +data \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..08d151d --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +default: help + +# Color variables +BOLD = \033[1m +PURPLE = \033[35m +GRAY = \033[37m +CYAN = \033[36m +NC = \033[0m + +.PHONY: help +help: + @echo "\nUsage: make [target] ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n" + @echo "Available targets:\n" + @echo "Development Environment ---------------------------------------------------" + @echo " ${BOLD}${PURPLE}clean-venv${NC} ${GRAY}(cv)${NC}: ${CYAN}Remove virtual environment.${NC}" + @echo " ${BOLD}${PURPLE}install${NC} ${GRAY}(i)${NC}: ${CYAN}Install package in development mode.${NC}" + @echo " ${BOLD}${PURPLE}venv${NC} ${GRAY}(v)${NC}: ${CYAN}Create virtual environment.${NC}\n" + @echo "Development Services -----------------------------------------------------" + @echo " ${BOLD}${PURPLE}dev-app${NC} ${GRAY}(da)${NC}: ${CYAN}Run only the prowes Flask app locally.${NC}" + @echo " ${BOLD}${PURPLE}dev-celery${NC} ${GRAY}(dc)${NC}: ${CYAN}Run only the Celery worker locally.${NC}" + @echo " ${BOLD}${PURPLE}dev-docker${NC} ${GRAY}(dd)${NC}: ${CYAN}Run all docker dev services.${NC}" + +# Development Environment +.PHONY: clean-venv cv +clean-venv cv: + @echo "Removing virtual environment..." + rm -rf .venv + +.PHONY: install i +install i: + @echo "Installing package in development mode..." + pip install -e . + +.PHONY: venv v +venv v: + @echo "Creating virtual environment..." + python -m venv .venv + @echo "Run 'source .venv/bin/activate' to activate the virtual environment" + +# Development Services +.PHONY: dev-docker dd +dev-docker dd: + @echo "Starting Flask app..." + docker compose -f docker-compose.dev.yaml up -d + +.PHONY: dev-app da +dev-app da: + @echo "Starting FOCA app..." + python pro_wes/app.py + +.PHONY: dev-celery dc +dev-celery dc: + @echo "Starting Celery worker..." + cd pro_wes && celery -A celery_worker worker -E --loglevel=info \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml new file mode 100644 index 0000000..58e8a57 --- /dev/null +++ b/docker-compose.dev.yaml @@ -0,0 +1,23 @@ +services: + rabbitmq: + image: rabbitmq:3-management + restart: unless-stopped + links: + - mongodb + ports: + - "5672:5672" + + mongodb: + image: mongo:3.2 + restart: unless-stopped + volumes: + - ${PROWES_DATA_DIR:-../data/pro_wes}/db:${PWD}/data/db + ports: + - "27017:27017" + + flower: + image: mher/flower:0.9.7 + restart: unless-stopped + command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555 + ports: + - "5555:5555" diff --git a/docker-compose.yaml b/docker-compose.yaml index 01d896f..5003b87 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '3.6' services: prowes-worker: @@ -22,7 +21,7 @@ services: restart: unless-stopped links: - mongodb - command: bash -c "cd /app/pro_wes; gunicorn -c gunicorn.py wsgi:app" + command: bash -c "cd /app; gunicorn -c pro_wes/gunicorn.py pro_wes.wsgi:app" ports: - "8090:8080" volumes: diff --git a/pro_wes/app.py b/pro_wes/app.py index 66aa05a..a6373c1 100644 --- a/pro_wes/app.py +++ b/pro_wes/app.py @@ -1,5 +1,6 @@ """proWES application entry point.""" +import os from pathlib import Path from connexion import App @@ -16,8 +17,13 @@ def init_app() -> App: Returns: FOCA application. """ + _parent_dir = Path(__file__).resolve().parent + if os.environ.get("ENVIRONMENT") == "DEV": + config_file = _parent_dir / "config.dev.yaml" + else: + config_file = _parent_dir / "config.yaml" foca = Foca( - config_file=Path(__file__).resolve().parent / "config.yaml", + config_file=config_file, custom_config_model="pro_wes.config_models.CustomConfig", ) app = foca.create_app() @@ -47,5 +53,6 @@ def run_app(app: App) -> None: if __name__ == "__main__": + os.environ.setdefault("ENVIRONMENT", "DEV") foca_app = init_app() run_app(app=foca_app) diff --git a/pro_wes/config.dev.yaml b/pro_wes/config.dev.yaml new file mode 100644 index 0000000..ff2e59a --- /dev/null +++ b/pro_wes/config.dev.yaml @@ -0,0 +1,154 @@ +# FOCA configuration +# Available in app context as attributes of `current_app.config.foca` +# Automatically validated via FOCA +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html + +# Server configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ServerConfig +server: + host: "0.0.0.0" + port: 8080 + debug: True + environment: development + testing: False + use_reloader: False + +# Security configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.SecurityConfig +security: + auth: + add_key_to_claims: True + algorithms: + - RS256 + allow_expired: False + audience: null + validation_methods: + - userinfo + - public_key + validation_checks: any + +# Database configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.DBConfig +db: + host: localhost + port: 27017 + dbs: + runStore: + collections: + runs: + indexes: + - keys: + run_id: 1 + task_id: 1 + options: + "unique": True + "sparse": True + service_info: + indexes: + - keys: + id: 1 + +# API configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.APIConfig +api: + specs: + - path: + - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml + - pro_wes/api/additions.openapi.yaml + add_operation_fields: + x-openapi-router-controller: pro_wes.ga4gh.wes.controllers + add_security_fields: + x-bearerInfoFunc: app.validate_token + disable_auth: True + connexion: + strict_validation: True + # workaround until cwl-WES responses are fixed + validate_responses: False + base_path: /ga4gh/wes/v1 + options: + swagger_ui: True + serve_spec: True + +# Logging configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.LogConfig +log: + version: 1 + disable_existing_loggers: False + formatters: + standard: + class: logging.Formatter + style: "{" + format: "[{asctime}: {levelname:<8}] {message} [{name}]" + handlers: + console: + class: logging.StreamHandler + level: 20 + formatter: standard + stream: ext://sys.stderr + root: + level: 10 + handlers: [console] + +# Background job configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.JobsConfig +jobs: + host: rabbitmq + port: 5672 + backend: "rpc://" + include: + - pro_wes.tasks.track_run_progress + +# Exception configuration +# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ExceptionConfig +exceptions: + required_members: [["message"], ["code"]] + status_member: ["code"] + exceptions: pro_wes.exceptions.exceptions + +# Custom configuration +# Available in app context as attributes of `current_app.config.foca` +custom: + defaults: + timeout: 2 + post_runs: + db_insert_attempts: 10 + id_charset: string.ascii_uppercase + string.digits + id_length: 6 + polling_attempts: 100 + polling_wait: 3 + storage_path: "./data" + timeout_job: null + timeout_post: null + list_runs: + default_page_size: 5 + service_info: + auth_instructions_url: "https://example.org/auth_instructions" + default_workflow_engine_parameters: [] + id: v1.wes.ga4gh.org.example + name: "proWES example deployment" + supported_filesystem_protocols: + - https + supported_wes_versions: + - 1.0.1 + tags: + key: "value" + type: + group: "org.ga4gh" + artifact: "wes" + version: "1.0.1" + workflow_engine_versions: + cwl-engine: "1.2.3" + workflow_type_versions: + CWL: + workflow_type_version: + - v1.0 + description: "WES gateway service" + organization: + name: "Example organization" + url: "https://example.org" + contactUrl: "support@example.org" + documentationUrl: "https://example.org/docs" + createdAt: "2020-01-01T00:00:00Z" + updatedAt: "2022-12-31T00:00:00Z" + environment: "test" + version: "0.18.0" diff --git a/pro_wes/config.yaml b/pro_wes/config.yaml index 834376f..febfd4c 100644 --- a/pro_wes/config.yaml +++ b/pro_wes/config.yaml @@ -53,8 +53,8 @@ db: api: specs: - path: - - api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml - - api/additions.openapi.yaml + - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml + - pro_wes/api/additions.openapi.yaml add_operation_fields: x-openapi-router-controller: pro_wes.ga4gh.wes.controllers add_security_fields: diff --git a/requirements.txt b/requirements.txt index 2d6ddc5..84a9ba1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ connexion<3 email-validator>=2.1.0,<3 foca>=0.12.1 gunicorn>=20.1.0,<21 +pymongo==4.7 # Cf. https://github.com/elixir-cloud-aai/foca/issues/246 From 9f5e99daa13ec32a6210f5c001bf1d1c9d0366e2 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 19 Dec 2024 15:56:02 +0530 Subject: [PATCH 2/6] make it run locally --- Makefile | 17 +++- docker-compose.dev.yaml | 2 +- docker-compose.yaml | 6 +- pro_wes/app.py | 9 +- pro_wes/config.dev.yaml | 154 ----------------------------- pro_wes/config.yaml | 6 +- pro_wes/ga4gh/wes/workflow_runs.py | 4 +- requirements.txt | 3 +- 8 files changed, 28 insertions(+), 173 deletions(-) delete mode 100644 pro_wes/config.dev.yaml diff --git a/Makefile b/Makefile index 08d151d..93e5a3c 100644 --- a/Makefile +++ b/Makefile @@ -51,4 +51,19 @@ dev-app da: .PHONY: dev-celery dc dev-celery dc: @echo "Starting Celery worker..." - cd pro_wes && celery -A celery_worker worker -E --loglevel=info \ No newline at end of file + cd pro_wes && celery -A celery_worker worker -E --loglevel=info + +.PHONY: lint fl +lint fl: + + @echo "1.Formatting with black..." + black --exclude .venv pro_wes/ setup.py tests/ + @echo "\n\n2.Checking style with flake8..." + black --exclude .venv pro_wes/ setup.py tests/ + @echo "\n\n3.Running pylint..." + pylint pro_wes/ setup.py + +.PHONY: type-check tc +type-check tc: + @echo "Running mypy..." + mypy pro_wes/ setup.py \ No newline at end of file diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 58e8a57..85111a9 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -11,7 +11,7 @@ services: image: mongo:3.2 restart: unless-stopped volumes: - - ${PROWES_DATA_DIR:-../data/pro_wes}/db:${PWD}/data/db + - ${PROWES_DATA_DIR:-../data/pro_wes}/db:/data/db ports: - "27017:27017" diff --git a/docker-compose.yaml b/docker-compose.yaml index 5003b87..50c7dc0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,7 +33,7 @@ services: links: - mongodb ports: - - "5682:5672" + - "5672:5672" mongodb: image: mongo:3.2 @@ -41,7 +41,7 @@ services: volumes: - ${PROWES_DATA_DIR:-../data/pro_wes}/db:/data/db ports: - - "27027:27017" + - "27017:27017" flower: image: mher/flower:0.9.7 @@ -50,4 +50,4 @@ services: - prowes-worker command: flower --broker=amqp://guest:guest@rabbitmq:5672// --port=5555 ports: - - "5565:5555" + - "5555:5555" diff --git a/pro_wes/app.py b/pro_wes/app.py index a6373c1..66aa05a 100644 --- a/pro_wes/app.py +++ b/pro_wes/app.py @@ -1,6 +1,5 @@ """proWES application entry point.""" -import os from pathlib import Path from connexion import App @@ -17,13 +16,8 @@ def init_app() -> App: Returns: FOCA application. """ - _parent_dir = Path(__file__).resolve().parent - if os.environ.get("ENVIRONMENT") == "DEV": - config_file = _parent_dir / "config.dev.yaml" - else: - config_file = _parent_dir / "config.yaml" foca = Foca( - config_file=config_file, + config_file=Path(__file__).resolve().parent / "config.yaml", custom_config_model="pro_wes.config_models.CustomConfig", ) app = foca.create_app() @@ -53,6 +47,5 @@ def run_app(app: App) -> None: if __name__ == "__main__": - os.environ.setdefault("ENVIRONMENT", "DEV") foca_app = init_app() run_app(app=foca_app) diff --git a/pro_wes/config.dev.yaml b/pro_wes/config.dev.yaml deleted file mode 100644 index ff2e59a..0000000 --- a/pro_wes/config.dev.yaml +++ /dev/null @@ -1,154 +0,0 @@ -# FOCA configuration -# Available in app context as attributes of `current_app.config.foca` -# Automatically validated via FOCA -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html - -# Server configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ServerConfig -server: - host: "0.0.0.0" - port: 8080 - debug: True - environment: development - testing: False - use_reloader: False - -# Security configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.SecurityConfig -security: - auth: - add_key_to_claims: True - algorithms: - - RS256 - allow_expired: False - audience: null - validation_methods: - - userinfo - - public_key - validation_checks: any - -# Database configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.DBConfig -db: - host: localhost - port: 27017 - dbs: - runStore: - collections: - runs: - indexes: - - keys: - run_id: 1 - task_id: 1 - options: - "unique": True - "sparse": True - service_info: - indexes: - - keys: - id: 1 - -# API configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.APIConfig -api: - specs: - - path: - - pro_wes/api/20201124.tag_1_0_1.workflow_execution_service.openapi.yaml - - pro_wes/api/additions.openapi.yaml - add_operation_fields: - x-openapi-router-controller: pro_wes.ga4gh.wes.controllers - add_security_fields: - x-bearerInfoFunc: app.validate_token - disable_auth: True - connexion: - strict_validation: True - # workaround until cwl-WES responses are fixed - validate_responses: False - base_path: /ga4gh/wes/v1 - options: - swagger_ui: True - serve_spec: True - -# Logging configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.LogConfig -log: - version: 1 - disable_existing_loggers: False - formatters: - standard: - class: logging.Formatter - style: "{" - format: "[{asctime}: {levelname:<8}] {message} [{name}]" - handlers: - console: - class: logging.StreamHandler - level: 20 - formatter: standard - stream: ext://sys.stderr - root: - level: 10 - handlers: [console] - -# Background job configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.JobsConfig -jobs: - host: rabbitmq - port: 5672 - backend: "rpc://" - include: - - pro_wes.tasks.track_run_progress - -# Exception configuration -# Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.ExceptionConfig -exceptions: - required_members: [["message"], ["code"]] - status_member: ["code"] - exceptions: pro_wes.exceptions.exceptions - -# Custom configuration -# Available in app context as attributes of `current_app.config.foca` -custom: - defaults: - timeout: 2 - post_runs: - db_insert_attempts: 10 - id_charset: string.ascii_uppercase + string.digits - id_length: 6 - polling_attempts: 100 - polling_wait: 3 - storage_path: "./data" - timeout_job: null - timeout_post: null - list_runs: - default_page_size: 5 - service_info: - auth_instructions_url: "https://example.org/auth_instructions" - default_workflow_engine_parameters: [] - id: v1.wes.ga4gh.org.example - name: "proWES example deployment" - supported_filesystem_protocols: - - https - supported_wes_versions: - - 1.0.1 - tags: - key: "value" - type: - group: "org.ga4gh" - artifact: "wes" - version: "1.0.1" - workflow_engine_versions: - cwl-engine: "1.2.3" - workflow_type_versions: - CWL: - workflow_type_version: - - v1.0 - description: "WES gateway service" - organization: - name: "Example organization" - url: "https://example.org" - contactUrl: "support@example.org" - documentationUrl: "https://example.org/docs" - createdAt: "2020-01-01T00:00:00Z" - updatedAt: "2022-12-31T00:00:00Z" - environment: "test" - version: "0.18.0" diff --git a/pro_wes/config.yaml b/pro_wes/config.yaml index febfd4c..a0ff875 100644 --- a/pro_wes/config.yaml +++ b/pro_wes/config.yaml @@ -30,7 +30,7 @@ security: # Database configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.DBConfig db: - host: mongodb + host: localhost port: 27017 dbs: runStore: @@ -92,7 +92,7 @@ log: # Background job configuration # Cf. https://foca.readthedocs.io/en/latest/modules/foca.models.html#foca.models.config.JobsConfig jobs: - host: rabbitmq + host: localhost port: 5672 backend: "rpc://" include: @@ -116,7 +116,7 @@ custom: id_length: 6 polling_attempts: 100 polling_wait: 3 - storage_path: "/data" + storage_path: "./data" timeout_job: null timeout_post: null list_runs: diff --git a/pro_wes/ga4gh/wes/workflow_runs.py b/pro_wes/ga4gh/wes/workflow_runs.py index d3fd27e..74b97a8 100644 --- a/pro_wes/ga4gh/wes/workflow_runs.py +++ b/pro_wes/ga4gh/wes/workflow_runs.py @@ -482,7 +482,9 @@ def _validate_run_request( """ dict_of_lists = form_data.to_dict(flat=False) # flatten single item lists - dict_atomic = {k: v[0] if len(v) == 1 else v for k, v in dict_of_lists.items()} + dict_atomic: dict = { + k: v[0] if len(v) == 1 else v for k, v in dict_of_lists.items() + } # remove 'workflow_attachment' field dict_atomic.pop("workflow_attachment", None) model_instance = RunRequest(**dict_atomic) diff --git a/requirements.txt b/requirements.txt index 84a9ba1..14f3624 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ connexion<3 email-validator>=2.1.0,<3 -foca>=0.12.1 +foca==0.12.1 gunicorn>=20.1.0,<21 -pymongo==4.7 # Cf. https://github.com/elixir-cloud-aai/foca/issues/246 From 690bf026f2e599bc8c2d9bf8eb1cb1376ba61c30 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 19 Dec 2024 15:57:18 +0530 Subject: [PATCH 3/6] docker-compose is depricated and docker compose is used. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b5d2c7..959e704 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: with: python-version: "3.10" - name: Deploy app - run: docker-compose up -d --build + run: docker compose up -d --build - name: Wait for app startup run: sleep 20 - name: Probe endpoint From 5ba3e98e2fc6eec649078ef14182701808bb9241 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 19 Dec 2024 16:02:38 +0530 Subject: [PATCH 4/6] upgrade python version --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 959e704..a7c74f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,11 +17,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install requirements run: | + pip install -e . pip install -r requirements_dev.txt - pip install . - name: Lint with Flake8 run: flake8 - name: Lint with Pylint From 67e959713ecf7960ef33e9addebcccbbb522aee4 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 19 Dec 2024 16:07:06 +0530 Subject: [PATCH 5/6] yq --- .github/workflows/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a7c74f2..454671b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,8 +20,8 @@ jobs: python-version: "3.11" - name: Install requirements run: | - pip install -e . pip install -r requirements_dev.txt + pip install -r requirements.txt - name: Lint with Flake8 run: flake8 - name: Lint with Pylint @@ -45,6 +45,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.10" + - name: Install yq + run: pip install yq + - name: Update configuration hosts + run: | + yq -y --in-place '.db.host = "mongodb"' pro_wes/config.yaml + yq -y --in-place '.jobs.host = "rabbitmq"' pro_wes/config.yaml - name: Deploy app run: docker compose up -d --build - name: Wait for app startup From e24b0dbe64762f795db4a7c84db6b4c156a3ac00 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 19 Dec 2024 16:14:35 +0530 Subject: [PATCH 6/6] revert pip chagens --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 454671b..c80235a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,11 +17,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.10" - name: Install requirements run: | pip install -r requirements_dev.txt - pip install -r requirements.txt + pip install . - name: Lint with Flake8 run: flake8 - name: Lint with Pylint