From 83f0c2de1a953515b8bbadd965b82d14a6261b1e Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 27 Jun 2024 13:25:42 +0530 Subject: [PATCH 01/11] fix: remove format warning --- docs/source/conf.py | 56 ++++++++-------- pyproject.toml | 29 ++++---- tesk/api/ga4gh/tes/controllers.py | 66 +++++++++--------- tesk/app.py | 68 +++++++++---------- tesk/exceptions.py | 108 +++++++++++++++--------------- 5 files changed, 162 insertions(+), 165 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cd84ae5c..2761d14e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,43 +12,43 @@ import tomli -sys.path.insert(0, os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath("../..")) # -- Project information ----------------------------------------------------- def _get_project_meta(): - _pyproject_path = Path(__file__).parents[2] / 'pyproject.toml' - with open(_pyproject_path, mode='rb') as pyproject: - return tomli.load(pyproject)['tool']['poetry'] + _pyproject_path = Path(__file__).parents[2] / "pyproject.toml" + with open(_pyproject_path, mode="rb") as pyproject: + return tomli.load(pyproject)["tool"]["poetry"] pkg_meta = _get_project_meta() current_year = datetime.datetime.now().year -project = str(pkg_meta['name']) +project = str(pkg_meta["name"]) project_copyright = f"{current_year}, {str(pkg_meta['authors'][0])}" -author = str(pkg_meta['authors'][0]) +author = str(pkg_meta["authors"][0]) -version = str(pkg_meta['version']) +version = str(pkg_meta["version"]) release = version # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'sphinx.ext.autosummary', - # Used to write beautiful docstrings: - 'sphinx.ext.napoleon', - # Used to include .md files: - 'm2r2', - # Used to insert typehints into the final docs: - 'sphinx_autodoc_typehints', - # Used to embed values from the source code into the docs: - 'added_value', + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.autosummary", + # Used to write beautiful docstrings: + "sphinx.ext.napoleon", + # Used to include .md files: + "m2r2", + # Used to insert typehints into the final docs: + "sphinx_autodoc_typehints", + # Used to embed values from the source code into the docs: + "added_value", ] # Set `typing.TYPE_CHECKING` to `True`: @@ -57,39 +57,39 @@ def _get_project_meta(): always_document_param_types = False # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: -source_suffix = ['.rst', '.md'] +source_suffix = [".rst", ".md"] # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = 'en' +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'furo' +html_theme = "furo" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # -- Extension configuration ------------------------------------------------- napoleon_numpy_docstring = False diff --git a/pyproject.toml b/pyproject.toml index f92be0a7..65ea0ed9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,9 +70,9 @@ types-urllib3 = "^1.26.25.14" types-werkzeug = "^1.0.9" [tool.poetry.scripts] -api = 'tesk.app:main' -filer = 'tesk.services.filer:main' -taskmaster = 'tesk.services.taskmaster:main' +api = "tesk.app:main" +filer = "tesk.services.filer:main" +taskmaster = "tesk.services.taskmaster:main" [tool.pytest_env] FTP_FIXTURE_SCOPE = "function" @@ -83,17 +83,20 @@ FTP_USER = "user" TESK_FTP_PASSWORD = "pass" TESK_FTP_USERNAME = "user" +[tool.ruff] +exclude = [ + "tesk/services/*", + "tests/test_unit/test_services/*", +] +indent-width = 2 + [tool.ruff.format] docstring-code-format = true -indent-style = "tab" -quote-style = "single" +indent-style = "space" +line-ending = "lf" +quote-style = "double" [tool.ruff.lint] -ignore = [ - "D203", # conflicts with D202 as formatter removes black line before class docstring - "D206", # conflicts with E101, `format.indent-style="tab"` - "D213", # conflicts with D212, would have been ignored regardless -] select = [ "B", # flake8-bugbear "D", # pydocstyle @@ -105,12 +108,8 @@ select = [ "UP", # pyupgrade ] -[tool.ruff.lint.per-file-ignores] # ignore all pydocstyle errors in test and services modules -"tesk/services/**/*.py" = ["D"] -"tests/test_unit/test_services/**/*.py" = ["D"] - [tool.ruff.lint.pydocstyle] convention = "google" [tool.typos.default.extend-words] -mke = 'mke' +mke = "mke" diff --git a/tesk/api/ga4gh/tes/controllers.py b/tesk/api/ga4gh/tes/controllers.py index 55609688..53d6b2c3 100644 --- a/tesk/api/ga4gh/tes/controllers.py +++ b/tesk/api/ga4gh/tes/controllers.py @@ -12,60 +12,60 @@ # POST /tasks/{id}:cancel @log_traffic def CancelTask(id, *args, **kwargs) -> dict: # type: ignore - """Cancel unfinished task. + """Cancel unfinished task. - Args: - id: Task identifier. - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + id: Task identifier. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # POST /tasks @log_traffic def CreateTask(*args, **kwargs) -> dict: # type: ignore - """Create task. + """Create task. - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # GET /tasks/service-info @log_traffic def GetServiceInfo(*args, **kwargs) -> dict: # type: ignore - """Get service info. + """Get service info. - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # GET /tasks @log_traffic def ListTasks(*args, **kwargs) -> dict: # type: ignore - """List all available tasks. + """List all available tasks. - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # GET /tasks @log_traffic def GetTask(*args, **kwargs) -> dict: # type: ignore - """Get info for individual task. - - Args: - id: Task identifier. - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + """Get info for individual task. + + Args: + id: Task identifier. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass diff --git a/tesk/app.py b/tesk/app.py index 52e3d026..6ace9684 100644 --- a/tesk/app.py +++ b/tesk/app.py @@ -11,43 +11,41 @@ def init_app() -> FlaskApp: - """Initialize and return the FOCA app. - - This function initializes the FOCA app by loading the configuration - from the environment variable `TESK_FOCA_CONFIG_PATH` if set, or from - the default path if not. It raises a `FileNotFoundError` if the - configuration file is not found. - - Returns: - FlaskApp: A Connexion application instance. - - Raises: - FileNotFoundError: If the configuration file is not found. - """ - # Determine the configuration path - config_path_env = os.getenv('TESK_FOCA_CONFIG_PATH') - if config_path_env: - config_path = Path(config_path_env).resolve() - else: - config_path = ( - Path(__file__).parents[1] / 'deployment' / 'config.yaml' - ).resolve() - - # Check if the configuration file exists - if not config_path.exists(): - raise FileNotFoundError(f'Config file not found at: {config_path}') - - foca = Foca( - config_file=config_path, - ) - return foca.create_app() + """Initialize and return the FOCA app. + + This function initializes the FOCA app by loading the configuration + from the environment variable `TESK_FOCA_CONFIG_PATH` if set, or from + the default path if not. It raises a `FileNotFoundError` if the + configuration file is not found. + + Returns: + FlaskApp: A Connexion application instance. + + Raises: + FileNotFoundError: If the configuration file is not found. + """ + # Determine the configuration path + config_path_env = os.getenv("TESK_FOCA_CONFIG_PATH") + if config_path_env: + config_path = Path(config_path_env).resolve() + else: + config_path = (Path(__file__).parents[1] / "deployment" / "config.yaml").resolve() + + # Check if the configuration file exists + if not config_path.exists(): + raise FileNotFoundError(f"Config file not found at: {config_path}") + + foca = Foca( + config_file=config_path, + ) + return foca.create_app() def main() -> None: - """Run FOCA application.""" - app = init_app() - app.run(port=app.port) + """Run FOCA application.""" + app = init_app() + app.run(port=app.port) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/tesk/exceptions.py b/tesk/exceptions.py index 793c0338..ffe58fd5 100644 --- a/tesk/exceptions.py +++ b/tesk/exceptions.py @@ -1,68 +1,68 @@ """App exceptions.""" from connexion.exceptions import ( - BadRequestProblem, - ExtraParameterProblem, - Forbidden, - OAuthProblem, - Unauthorized, + BadRequestProblem, + ExtraParameterProblem, + Forbidden, + OAuthProblem, + Unauthorized, ) from werkzeug.exceptions import ( - BadRequest, - InternalServerError, - NotFound, + BadRequest, + InternalServerError, + NotFound, ) # exceptions raised in app context exceptions = { - Exception: { - 'title': 'Unknown error', - 'detail': 'An unexpected error occurred.', - 'status': 500, - }, - BadRequestProblem: { - 'title': 'Bad request', - 'detail': 'The request is malformed', - 'status': 400, - }, - BadRequest: { - 'title': 'Bad request', - 'detail': 'The request is malformed.', - 'status': 400, - }, - ExtraParameterProblem: { - 'title': 'Bad request', - 'detail': 'The request is malformed.', - 'status': 400, - }, - Unauthorized: { - 'title': 'Unauthorized', - 'detail': 'The request is unauthorized.', - 'status': 401, - }, - OAuthProblem: { - 'title': 'Unauthorized', - 'detail': 'The request is unauthorized.', - 'status': 401, - }, - Forbidden: { - 'title': 'Forbidden', - 'detail': 'The requester is not authorized to perform this action.', - 'status': 403, - }, - NotFound: { - 'title': 'Not found', - 'detail': "The requested resource wasn't found.", - 'status': 404, - }, - InternalServerError: { - 'title': 'Internal server error', - 'detail': 'An unexpected error occurred.', - 'status': 500, - }, + Exception: { + "title": "Unknown error", + "detail": "An unexpected error occurred.", + "status": 500, + }, + BadRequestProblem: { + "title": "Bad request", + "detail": "The request is malformed", + "status": 400, + }, + BadRequest: { + "title": "Bad request", + "detail": "The request is malformed.", + "status": 400, + }, + ExtraParameterProblem: { + "title": "Bad request", + "detail": "The request is malformed.", + "status": 400, + }, + Unauthorized: { + "title": "Unauthorized", + "detail": "The request is unauthorized.", + "status": 401, + }, + OAuthProblem: { + "title": "Unauthorized", + "detail": "The request is unauthorized.", + "status": 401, + }, + Forbidden: { + "title": "Forbidden", + "detail": "The requester is not authorized to perform this action.", + "status": 403, + }, + NotFound: { + "title": "Not found", + "detail": "The requested resource wasn't found.", + "status": 404, + }, + InternalServerError: { + "title": "Internal server error", + "detail": "An unexpected error occurred.", + "status": 500, + }, } # exceptions raised outside of app context class ValidationError(Exception): - """Value or object is not compatible with required type or schema.""" + """Value or object is not compatible with required type or schema.""" From 64fcd81067bd5fd9ff31995a980ea9f45e6e2b22 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 30 Jun 2024 11:01:24 +0530 Subject: [PATCH 02/11] chore: add models --- tesk/api/ga4gh/tes/models.py | 412 +++++++++++++++++++++++++++++++++++ 1 file changed, 412 insertions(+) create mode 100644 tesk/api/ga4gh/tes/models.py diff --git a/tesk/api/ga4gh/tes/models.py b/tesk/api/ga4gh/tes/models.py new file mode 100644 index 00000000..88360152 --- /dev/null +++ b/tesk/api/ga4gh/tes/models.py @@ -0,0 +1,412 @@ +"""Models for TES API.""" + +from enum import Enum +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field + + +class TesCancelTaskResponse(BaseModel): + """Represents a response for task cancellation in TES.""" + + +class TesCreateTaskResponse(BaseModel): + """Represents a response for task creation in TES. + + Attributes: + id (str): Task identifier assigned by the server. + """ + + id: str = Field(..., description="Task identifier assigned by the server.") + + +class TesExecutor(BaseModel): + """Represents an executor configuration for TES tasks. + + Attributes: + image (str): Name of the container image. + command (List[str]): Sequence of program arguments to execute. + workdir (Optional[str]): Working directory for the command. + stdin (Optional[str]): Path to file for executor's stdin. + stdout (Optional[str]): Path to file for executor's stdout. + stderr (Optional[str]): Path to file for executor's stderr. + env (Optional[Dict[str, str]]): Environmental variables for the container. + ignore_error (Optional[bool]): Flag to continue on errors. + """ + + image: str = Field( + ..., + description="Name of the container image. The string will be passed as the image\n" + "argument to the containerization run command. Examples:\n" + " - `ubuntu`\n" + " - `quay.io/aptible/ubuntu`\n" + " - `gcr.io/my-org/my-image`\n" + " - `myregistryhost:5000/fedora/httpd:version1.0`", + example="ubuntu:20.04", + ) + command: list[str] = Field( + ..., + description="A sequence of program arguments to execute, where the first argument\n" + "is the program to execute (i.e. argv). Example:\n" + "```\n" + "{\n" + ' "command" : ["/bin/md5", "/data/file1"]\n' + "}\n" + "```", + example=["/bin/md5", "/data/file1"], + ) + workdir: Optional[str] = Field( + None, + description="The working directory that the command will be executed in.\n" + "If not defined, the system will default to the directory set by\n" + "the container image.", + example="/data/", + ) + stdin: Optional[str] = Field( + None, + description="Path inside the container to a file which will be piped\n" + "to the executor's stdin. This must be an absolute path. This mechanism\n" + "could be used in conjunction with the input declaration to process\n" + "a data file using a tool that expects STDIN.\n\n" + "For example, to get the MD5 sum of a file by reading it into the STDIN\n" + "```\n" + "{\n" + ' "command" : ["/bin/md5"],\n' + ' "stdin" : "/data/file1"\n' + "}\n" + "```", + example="/data/file1", + ) + stdout: Optional[str] = Field( + None, + description="Path inside the container to a file where the executor's\n" + "stdout will be written to. Must be an absolute path. Example:\n" + "```\n" + "{\n" + ' "stdout" : "/tmp/stdout.log"\n' + "}\n" + "```", + example="/tmp/stdout.log", + ) + stderr: Optional[str] = Field( + None, + description="Path inside the container to a file where the executor's\n" + "stderr will be written to. Must be an absolute path. Example:\n" + "```\n" + "{\n" + ' "stderr" : "/tmp/stderr.log"\n' + "}\n" + "```", + example="/tmp/stderr.log", + ) + env: Optional[Dict[str, str]] = Field( + None, + description="Environmental variables to set within the container. Example:\n" + "```\n" + "{\n" + ' "env" : {\n' + ' "ENV_CONFIG_PATH" : "/data/config.file",\n' + ' "BLASTDB" : "/data/GRC38",\n' + ' "HMMERDB" : "/data/hmmer"\n' + " }\n" + "}\n" + "```", + example={"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}, + ) + ignore_error: Optional[bool] = Field( + None, + description="Default behavior of running an array of executors is that execution\n" + "stops on the first error. If `ignore_error` is `True`, then the\n" + "runner will record error exit codes, but will continue on to the next\n" + "tesExecutor.", + ) + + +class TesExecutorLog(BaseModel): + """Represents the log of an executor in TES. + + Attributes: + start_time (Optional[str]): Start time of the executor. + end_time (Optional[str]): End time of the executor. + stdout (Optional[str]): Stdout content. + stderr (Optional[str]): Stderr content. + exit_code (int): Exit code. + """ + + start_time: Optional[str] = Field( + None, + description="Time the executor started, in RFC 3339 format.", + example="2020-10-02T10:00:00-05:00", + ) + end_time: Optional[str] = Field( + None, + description="Time the executor ended, in RFC 3339 format.", + example="2020-10-02T11:00:00-05:00", + ) + stdout: Optional[str] = Field( + None, + description="Stdout content.\n\nThis is meant for convenience. No guarantees are " + "made about the content.\n Implementations may chose different approaches: only " + "the head, only the tail,\n a URL reference only, etc.\n\nIn order to capture the " + "full stdout client should set Executor.stdout\n to a container file path, and " + "use Task.outputs to upload that file\n to permanent storage.", + ) + stderr: Optional[str] = Field( + None, + description="Stderr content.\n\nThis is meant for convenience. No guarantees are " + "made about the content.\n Implementations may chose different approaches: only " + "the head, only the tail,\n a URL reference only, etc.\n\nIn order to capture the " + "full stderr client should set Executor.stderr\n to a container file path, and " + "use Task.outputs to upload that file\n to permanent storage.", + ) + exit_code: int = Field(..., description="Exit code.") + + +class TesFileType(Enum): + """Enum for TES file types.""" + + FILE = "FILE" + DIRECTORY = "DIRECTORY" + + +class TesInput(BaseModel): + """Represents an input file for a TES task. + + Attributes: + name (Optional[str]): Name of the input file. + description (Optional[str]): Description of the input file. + url (Optional[str]): URL in long term storage. + path (str): Path of the file inside the container. + type (Optional[TesFileType]): Type of the file. + content (Optional[str]): File content literal. + streamable (Optional[bool]): Flag for streaming interface. + """ + + name: Optional[str] = None + description: Optional[str] = None + url: Optional[str] = Field( + None, + description='REQUIRED, unless "content" is set.\n\n' + "URL in long term storage, for example:\n" + " - s3://my-object-store/file1\n" + " - gs://my-bucket/file2\n" + " - file:///path/to/my/file\n" + " - /path/to/my/file", + example="s3://my-object-store/file1", + ) + path: str = Field( + ..., + description="Path of the file inside the container.\n" "Must be an absolute path.", + example="/data/file1", + ) + type: Optional[TesFileType] = "FILE" + content: Optional[str] = Field( + None, + description="File content literal.\n\n" + "Implementations should support a minimum of 128 KiB in this field\n" + "and may define their own maximum.\n\n" + "UTF-8 encoded\n\n" + 'If content is not empty, "url" must be ignored.', + ) + streamable: Optional[bool] = Field( + None, + description="Indicate that a file resource could be accessed using a streaming\n" + "interface, ie a FUSE mounted s3 object. This flag indicates that\n" + "using a streaming mount, as opposed to downloading the whole file to\n" + "the local scratch space, may be faster despite the latency and\n" + "overhead. This does not mean that the backend will use a streaming\n" + "interface, as it may not be provided by the vendor, but if the\n" + "capacity is available it can be used without degrading the\n" + "performance of the underlying program.", + ) + + +class TesOutput(BaseModel): + """Represents an output file for a TES task. + + Attributes: + name (Optional[str]): Name of the output file. + description (Optional[str]): Description of the output file. + url (Optional[str]): URL in long term storage. + path (str): Path of the file inside the container. + type (Optional[TesFileType]): Type of the file. + """ + + name: Optional[str] = None + description: Optional[str] = None + url: Optional[str] = Field( + None, + description='REQUIRED, unless "content" is set.\n\n' + "URL in long term storage, for example:\n" + " - s3://my-object-store/file1\n" + " - gs://my-bucket/file2\n" + " - file:///path/to/my/file\n" + " - /path/to/my/file", + example="s3://my-object-store/file1", + ) + path: str = Field( + ..., + description="Path of the file inside the container.\n" "Must be an absolute path.", + example="/data/file1", + ) + type: Optional[TesFileType] = "FILE" + + +class TesLogs(BaseModel): + """Represents the logs for a TES task. + + Attributes: + logs (Optional[List[TesExecutorLog]]): Logs for the task. + """ + + logs: Optional[List[TesExecutorLog]] = Field( + None, + description="Executor logs.\n\n" + "If no logs are available, it should be set to an empty array.", + ) + + +class TesResources(BaseModel): + """Represents resource requirements for a TES task. + + Attributes: + cpu_cores (Optional[int]): Number of CPU cores. + preemptible (Optional[bool]): Flag for preemptible resources. + ram_gb (Optional[float]): Amount of RAM in GB. + disk_gb (Optional[float]): Amount of disk space in GB. + zones (Optional[List[str]]): Compute zones. + backend_parameters (Optional[Dict[str, str]]): Additional backend parameters. + """ + + cpu_cores: Optional[int] = Field( + None, + description="Number of CPU cores.", + example=4, + ) + preemptible: Optional[bool] = Field( + None, + description="Flag for preemptible resources.\n" + "If true, indicates that the work does not need to complete,\n" + "but it would be nice if it could.", + ) + ram_gb: Optional[float] = Field( + None, + description="Amount of RAM in GB.", + example=2.0, + ) + disk_gb: Optional[float] = Field( + None, + description="Amount of disk space in GB.", + example=10.0, + ) + zones: Optional[List[str]] = Field( + None, + description='Compute zones, e.g. Amazon EC2 "us-east-1a", GCE "us-central1".', + example=["us-east-1a", "us-central1"], + ) + backend_parameters: Optional[Dict[str, str]] = Field( + None, + description="Arbitrary parameters to pass to the backend. Must be\n" + "JSON-encodable.", + ) + + +class TesState(Enum): + """Enum for TES task states.""" + + UNKNOWN = "UNKNOWN" + QUEUED = "QUEUED" + INITIALIZING = "INITIALIZING" + RUNNING = "RUNNING" + PAUSED = "PAUSED" + COMPLETE = "COMPLETE" + CANCELED = "CANCELED" + EXECUTOR_ERROR = "EXECUTOR_ERROR" + SYSTEM_ERROR = "SYSTEM_ERROR" + + +class TesTask(BaseModel): + """Represents a TES task. + + Attributes: + id (Optional[str]): Task identifier. + state (Optional[TesState]): State of the task. + name (Optional[str]): Name of the task. + description (Optional[str]): Description of the task. + executors (List[TesExecutor]): Executors for the task. + inputs (Optional[List[TesInput]]): Input files. + outputs (Optional[List[TesOutput]]): Output files. + resources (Optional[TesResources]): Resource requirements. + creation_time (Optional[str]): Creation time. + start_time (Optional[str]): Start time. + end_time (Optional[str]): End time. + logs (Optional[List[TesLogs]]): Logs for the task. + """ + + id: Optional[str] = Field( + None, + description="Task identifier, assigned by the server.", + ) + state: Optional[TesState] = Field( + TesState.UNKNOWN, + description="Task state.", + ) + name: Optional[str] = Field( + None, + description="Optional, must be a non-empty string.", + example="Example Task", + ) + description: Optional[str] = Field( + None, + description="Optional, must be a non-empty string.", + ) + executors: List[TesExecutor] = Field( + ..., + description="At least one Executor must be present.", + ) + inputs: Optional[List[TesInput]] = Field( + None, + description="Array of input files.", + ) + outputs: Optional[List[TesOutput]] = Field( + None, + description="Array of output files.", + ) + resources: Optional[TesResources] = Field( + None, + description="Resource requirements.", + ) + creation_time: Optional[str] = Field( + None, + description="Time the task was created, in RFC 3339 format.", + example="2020-10-02T10:00:00-05:00", + ) + start_time: Optional[str] = Field( + None, + description="Time the task was started, in RFC 3339 format.", + example="2020-10-02T10:05:00-05:00", + ) + end_time: Optional[str] = Field( + None, + description="Time the task ended, in RFC 3339 format.", + example="2020-10-02T11:00:00-05:00", + ) + logs: Optional[List[TesLogs]] = Field( + None, + description="Task logs.\n\n" + "If no logs are available, it should be set to an empty array.", + ) + + +class TesTaskLog(BaseModel): + """Represents the log for a TES task. + + Attributes: + logs (Optional[List[TesExecutorLog]]): Executor logs. + """ + + logs: Optional[List[TesExecutorLog]] = Field( + None, + description="Executor logs.\n\n" + "If no logs are available, it should be set to an empty array.", + ) From b140dd137a2658a5a931bfb520d898939fa22107 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 30 Jun 2024 11:46:20 +0530 Subject: [PATCH 03/11] chore: add models --- tesk/api/ga4gh/tes/models.py | 681 +++++++++++++++++++++-------------- 1 file changed, 416 insertions(+), 265 deletions(-) diff --git a/tesk/api/ga4gh/tes/models.py b/tesk/api/ga4gh/tes/models.py index 88360152..b4a70072 100644 --- a/tesk/api/ga4gh/tes/models.py +++ b/tesk/api/ga4gh/tes/models.py @@ -1,318 +1,401 @@ """Models for TES API.""" +from datetime import datetime from enum import Enum from typing import Dict, List, Optional -from pydantic import BaseModel, Field +from pydantic import AnyUrl, BaseModel, Field class TesCancelTaskResponse(BaseModel): - """Represents a response for task cancellation in TES.""" + """CancelTaskResponse describes a response from the CancelTask endpoint.""" + + pass class TesCreateTaskResponse(BaseModel): - """Represents a response for task creation in TES. + """CreateTaskResponse describes a response from the CreateTask endpoint. - Attributes: - id (str): Task identifier assigned by the server. + It will include the task ID that can be used to look up the status of the job. """ id: str = Field(..., description="Task identifier assigned by the server.") class TesExecutor(BaseModel): - """Represents an executor configuration for TES tasks. - - Attributes: - image (str): Name of the container image. - command (List[str]): Sequence of program arguments to execute. - workdir (Optional[str]): Working directory for the command. - stdin (Optional[str]): Path to file for executor's stdin. - stdout (Optional[str]): Path to file for executor's stdout. - stderr (Optional[str]): Path to file for executor's stderr. - env (Optional[Dict[str, str]]): Environmental variables for the container. - ignore_error (Optional[bool]): Flag to continue on errors. - """ + """An executor is a command to be run in a container.""" image: str = Field( ..., - description="Name of the container image. The string will be passed as the image\n" - "argument to the containerization run command. Examples:\n" - " - `ubuntu`\n" - " - `quay.io/aptible/ubuntu`\n" - " - `gcr.io/my-org/my-image`\n" - " - `myregistryhost:5000/fedora/httpd:version1.0`", + description=( + "Name of the container image. The string will be passed as the image" + "\nargument to the containerization run command. Examples:\n - `ubuntu`\n - " + "`quay.io/aptible/ubuntu`\n - `gcr.io/my-org/my-image`\n - " + "`myregistryhost:5000/fedora/httpd:version1.0`" + ), example="ubuntu:20.04", ) - command: list[str] = Field( + command: List[str] = Field( ..., description="A sequence of program arguments to execute, where the first argument\n" - "is the program to execute (i.e. argv). Example:\n" - "```\n" - "{\n" - ' "command" : ["/bin/md5", "/data/file1"]\n' - "}\n" - "```", + 'is the program to execute (i.e. argv). Example:\n```\n{\n "command" : ["/bin/md5"' + ', "/data/file1"]\n}\n```', example=["/bin/md5", "/data/file1"], ) workdir: Optional[str] = Field( - None, - description="The working directory that the command will be executed in.\n" - "If not defined, the system will default to the directory set by\n" - "the container image.", + default=None, + description=( + "The working directory that the command will be executed in.\nIf not " + "defined, the system will default to the directory set by\nthe container image." + ), example="/data/", ) stdin: Optional[str] = Field( - None, - description="Path inside the container to a file which will be piped\n" - "to the executor's stdin. This must be an absolute path. This mechanism\n" - "could be used in conjunction with the input declaration to process\n" - "a data file using a tool that expects STDIN.\n\n" - "For example, to get the MD5 sum of a file by reading it into the STDIN\n" - "```\n" - "{\n" - ' "command" : ["/bin/md5"],\n' - ' "stdin" : "/data/file1"\n' - "}\n" - "```", + default=None, + description=( + "Path inside the container to a file which will be piped\nto the " + "executor's stdin. This must be an absolute path. This mechanism" + "\ncould be used in conjunction with the input declaration to " + "process\na data file using a tool that expects STDIN.\n\nFor " + "example, to get the MD5 sum of a file by reading it into the " + 'STDIN\n```\n{\n "command" : ["/bin/md5"],\n "stdin" : "/data/file1"' + "\n}\n```" + ), example="/data/file1", ) stdout: Optional[str] = Field( - None, - description="Path inside the container to a file where the executor's\n" - "stdout will be written to. Must be an absolute path. Example:\n" - "```\n" - "{\n" - ' "stdout" : "/tmp/stdout.log"\n' - "}\n" - "```", + default=None, + description="Path inside the container to a file where the executor's\nstdout will " + 'be written to. Must be an absolute path. Example:\n```\n{\n "stdout" : ' + '"/tmp/stdout.log"\n}\n```', example="/tmp/stdout.log", ) stderr: Optional[str] = Field( - None, - description="Path inside the container to a file where the executor's\n" - "stderr will be written to. Must be an absolute path. Example:\n" - "```\n" - "{\n" - ' "stderr" : "/tmp/stderr.log"\n' - "}\n" - "```", + default=None, + description="Path inside the container to a file where the executor's\nstderr " + 'will be written to. Must be an absolute path. Example:\n```\n{\n "stderr" : ' + '"/tmp/stderr.log"\n}\n```', example="/tmp/stderr.log", ) env: Optional[Dict[str, str]] = Field( - None, - description="Environmental variables to set within the container. Example:\n" - "```\n" - "{\n" - ' "env" : {\n' - ' "ENV_CONFIG_PATH" : "/data/config.file",\n' - ' "BLASTDB" : "/data/GRC38",\n' - ' "HMMERDB" : "/data/hmmer"\n' - " }\n" - "}\n" - "```", + default=None, + description="Environmental variables to set within the container. Example:\n```\n" + '{\n "env" : {\n "ENV_CONFIG_PATH" : "/data/config.file",\n "BLASTDB" : ' + '"/data/GRC38",\n "HMMERDB" : "/data/hmmer"\n }\n}\n```', example={"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}, ) ignore_error: Optional[bool] = Field( - None, - description="Default behavior of running an array of executors is that execution\n" - "stops on the first error. If `ignore_error` is `True`, then the\n" - "runner will record error exit codes, but will continue on to the next\n" - "tesExecutor.", + default=None, + description="Default behavior of running an array of executors is that " + "execution\nstops on the first error. If `ignore_error` is `True`, then " + "the\nrunner will record error exit codes, but will continue on to the " + "next\ntesExecutor.", ) class TesExecutorLog(BaseModel): - """Represents the log of an executor in TES. - - Attributes: - start_time (Optional[str]): Start time of the executor. - end_time (Optional[str]): End time of the executor. - stdout (Optional[str]): Stdout content. - stderr (Optional[str]): Stderr content. - exit_code (int): Exit code. - """ + """ExecutorLog describes logging information related to an Executor.""" start_time: Optional[str] = Field( - None, + default=None, description="Time the executor started, in RFC 3339 format.", example="2020-10-02T10:00:00-05:00", ) end_time: Optional[str] = Field( - None, + default=None, description="Time the executor ended, in RFC 3339 format.", example="2020-10-02T11:00:00-05:00", ) stdout: Optional[str] = Field( - None, + default=None, description="Stdout content.\n\nThis is meant for convenience. No guarantees are " - "made about the content.\n Implementations may chose different approaches: only " - "the head, only the tail,\n a URL reference only, etc.\n\nIn order to capture the " - "full stdout client should set Executor.stdout\n to a container file path, and " - "use Task.outputs to upload that file\n to permanent storage.", + "made about the content.\nImplementations may chose different approaches: only the " + "head, only the tail,\na URL reference only, etc.\n\nIn order to capture the full " + "stdout client should set Executor.stdout\nto a container file path, and use Task." + "outputs to upload that file\nto permanent storage.", ) stderr: Optional[str] = Field( - None, + default=None, description="Stderr content.\n\nThis is meant for convenience. No guarantees are " - "made about the content.\n Implementations may chose different approaches: only " - "the head, only the tail,\n a URL reference only, etc.\n\nIn order to capture the " - "full stderr client should set Executor.stderr\n to a container file path, and " - "use Task.outputs to upload that file\n to permanent storage.", + "made about the content.\nImplementations may chose different approaches: only the " + "head, only the tail,\na URL reference only, etc.\n\nIn order to capture the full " + "stderr client should set Executor.stderr\nto a container file path, and use Task." + "outputs to upload that file\nto permanent storage.", ) exit_code: int = Field(..., description="Exit code.") class TesFileType(Enum): - """Enum for TES file types.""" + """Define if input/output element is a file or a directory. + + It is not required that the user provide this value, but it is required that + the server fill in the value once the information is available at run time. + """ FILE = "FILE" DIRECTORY = "DIRECTORY" class TesInput(BaseModel): - """Represents an input file for a TES task. - - Attributes: - name (Optional[str]): Name of the input file. - description (Optional[str]): Description of the input file. - url (Optional[str]): URL in long term storage. - path (str): Path of the file inside the container. - type (Optional[TesFileType]): Type of the file. - content (Optional[str]): File content literal. - streamable (Optional[bool]): Flag for streaming interface. - """ + """Input describes Task input files.""" name: Optional[str] = None description: Optional[str] = None url: Optional[str] = Field( - None, - description='REQUIRED, unless "content" is set.\n\n' - "URL in long term storage, for example:\n" - " - s3://my-object-store/file1\n" - " - gs://my-bucket/file2\n" - " - file:///path/to/my/file\n" - " - /path/to/my/file", + default=None, + description='REQUIRED, unless "content" is set.\n\nURL in long term storage, for ' + "example:\n - s3://my-object-store/file1\n - gs://my-bucket/file2\n - " + "file:///path/to/my/file\n - /path/to/my/file", example="s3://my-object-store/file1", ) path: str = Field( ..., - description="Path of the file inside the container.\n" "Must be an absolute path.", + description="Path of the file inside the container.\nMust be an absolute path.", example="/data/file1", ) - type: Optional[TesFileType] = "FILE" + type: Optional[TesFileType] = TesFileType.FILE content: Optional[str] = Field( - None, - description="File content literal.\n\n" - "Implementations should support a minimum of 128 KiB in this field\n" - "and may define their own maximum.\n\n" - "UTF-8 encoded\n\n" - 'If content is not empty, "url" must be ignored.', + default=None, + description="File content literal.\n\nImplementations should support a minimum of " + "128 KiB in this field\nand may define their own maximum.\n\nUTF-8 encoded\n\nIf " + 'content is not empty, "url" must be ignored.', ) streamable: Optional[bool] = Field( - None, - description="Indicate that a file resource could be accessed using a streaming\n" - "interface, ie a FUSE mounted s3 object. This flag indicates that\n" - "using a streaming mount, as opposed to downloading the whole file to\n" - "the local scratch space, may be faster despite the latency and\n" - "overhead. This does not mean that the backend will use a streaming\n" - "interface, as it may not be provided by the vendor, but if the\n" - "capacity is available it can be used without degrading the\n" - "performance of the underlying program.", + default=None, + description="Indicate that a file resource could be accessed using a streaming" + "\ninterface, ie a FUSE mounted s3 object. This flag indicates that\nusing a " + "streaming mount, as opposed to downloading the whole file to\nthe local scratch " + "space, may be faster despite the latency and\noverhead. This does not mean that " + "the backend will use a streaming\ninterface, as it may not be provided by the " + "vendor, but if the\ncapacity is available it can be used without degrading " + "the\nperformance of the underlying program.", ) class TesOutput(BaseModel): - """Represents an output file for a TES task. - - Attributes: - name (Optional[str]): Name of the output file. - description (Optional[str]): Description of the output file. - url (Optional[str]): URL in long term storage. - path (str): Path of the file inside the container. - type (Optional[TesFileType]): Type of the file. - """ + """Output describes Task output files.""" - name: Optional[str] = None - description: Optional[str] = None - url: Optional[str] = Field( - None, - description='REQUIRED, unless "content" is set.\n\n' - "URL in long term storage, for example:\n" - " - s3://my-object-store/file1\n" - " - gs://my-bucket/file2\n" - " - file:///path/to/my/file\n" - " - /path/to/my/file", - example="s3://my-object-store/file1", + name: Optional[str] = Field(None, description="User-provided name of output file") + description: Optional[str] = Field( + default=None, + description="Optional users provided description field, can be used for " + "documentation.", + ) + url: str = Field( + ..., + description="URL at which the TES server makes the output accessible after the " + "task is complete.\nWhen tesOutput.path contains wildcards, it must be a " + "directory; see\n`tesOutput.path_prefix` for details on how output URLs are " + "constructed in this case.\nFor Example:\n - `s3://my-object-store/file1`\n - " + "`gs://my-bucket/file2`\n - `file:///path/to/my/file`", ) path: str = Field( ..., - description="Path of the file inside the container.\n" "Must be an absolute path.", - example="/data/file1", + description="Absolute path of the file inside the container.\nMay contain pattern " + "matching wildcards to select multiple outputs at once, but mind\nimplications for " + "`tesOutput.url` and `tesOutput.path_prefix`.\nOnly wildcards defined in IEEE Std " + "1003.1-2017 (POSIX), 12.3 are supported; " + "see\nhttps://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13", ) - type: Optional[TesFileType] = "FILE" + path_prefix: Optional[str] = Field( + default=None, + description="Prefix to be removed from matching outputs if `tesOutput.path` " + "contains wildcards;\noutput URLs are constructed by appending pruned paths to the " + "directory specified\nin `tesOutput.url`.\nRequired if `tesOutput.path` contains " + "wildcards, ignored otherwise.", + ) + type: Optional[TesFileType] = TesFileType.FILE -class TesLogs(BaseModel): - """Represents the logs for a TES task. +class TesOutputFileLog(BaseModel): + """OutputFileLog describes a single output file. - Attributes: - logs (Optional[List[TesExecutorLog]]): Logs for the task. + This describes file details after the task has completed successfully, for logging + purposes. """ - logs: Optional[List[TesExecutorLog]] = Field( - None, - description="Executor logs.\n\n" - "If no logs are available, it should be set to an empty array.", + url: str = Field( + ..., description="URL of the file in storage, e.g. s3://bucket/file.txt" + ) + path: str = Field( + ..., + description="Path of the file inside the container. Must be an absolute path.", + ) + size_bytes: str = Field( + ..., + description="Size of the file in bytes. Note, this is currently coded as a " + "string\nbecause official JSON doesn't support int64 numbers.", + example=["1024"], ) class TesResources(BaseModel): - """Represents resource requirements for a TES task. - - Attributes: - cpu_cores (Optional[int]): Number of CPU cores. - preemptible (Optional[bool]): Flag for preemptible resources. - ram_gb (Optional[float]): Amount of RAM in GB. - disk_gb (Optional[float]): Amount of disk space in GB. - zones (Optional[List[str]]): Compute zones. - backend_parameters (Optional[Dict[str, str]]): Additional backend parameters. - """ + """Resources describes the resources requested by a task.""" cpu_cores: Optional[int] = Field( - None, - description="Number of CPU cores.", - example=4, + default=None, description="Requested number of CPUs", example=4 ) preemptible: Optional[bool] = Field( - None, - description="Flag for preemptible resources.\n" - "If true, indicates that the work does not need to complete,\n" - "but it would be nice if it could.", + default=None, + description="Define if the task is allowed to run on preemptible compute instances," + "\nfor example, AWS Spot. This option may have no effect when utilized\non some " + "backends that don't have the concept of preemptible jobs.", + example=False, ) ram_gb: Optional[float] = Field( - None, - description="Amount of RAM in GB.", - example=2.0, + default=None, description="Requested RAM required in gigabytes (GB)", example=8 ) disk_gb: Optional[float] = Field( - None, - description="Amount of disk space in GB.", - example=10.0, + default=None, description="Requested disk size in gigabytes (GB)", example=40 ) zones: Optional[List[str]] = Field( - None, - description='Compute zones, e.g. Amazon EC2 "us-east-1a", GCE "us-central1".', - example=["us-east-1a", "us-central1"], + default=None, + description="Request that the task be run in these compute zones. How this string\n" + "is utilized will be dependent on the backend system. For example, a\nsystem based " + "on a cluster queueing system may use this string to define\npriority queue to " + "which the job is assigned.", + example="us-west-1", ) backend_parameters: Optional[Dict[str, str]] = Field( - None, - description="Arbitrary parameters to pass to the backend. Must be\n" - "JSON-encodable.", + default=None, + description="Key/value pairs for backend configuration.\nServiceInfo shall return " + "a list of keys that a backend supports.\nKeys are case insensitive.\nIt is " + "expected that clients pass all runtime or hardware requirement key/values\nthat " + "are not mapped to existing tesResources properties to backend_parameters.\n" + "Backends shall log system warnings if a key is passed that is unsupported.\n" + "Backends shall not store or return unsupported keys if included in a task.\nIf " + "backend_parameters_strict equals true,\nbackends should fail the task if any " + "key/values are unsupported, otherwise,\nbackends should attempt to run the " + "task\nIntended uses include VM size selection, coprocessor configuration, etc." + '\nExample:\n```\n{\n "backend_parameters" : {\n "VmSize" : "Standard_D64_v3"\n' + " }\n}\n```", + example={"VmSize": "Standard_D64_v3"}, + ) + backend_parameters_strict: Optional[bool] = Field( + False, + description="If set to true, backends should fail the task if any " + "backend_parameters\nkey/values are unsupported, otherwise, backends should " + "attempt to run the task", + example=False, + ) + + +class Artifact(Enum): + """Artifact type.""" + + tes = "tes" + + +class ServiceType(BaseModel): + """Type of a GA4GH service.""" + + group: str = Field( + ..., + description="Namespace in reverse domain name format. Use `org.ga4gh` for " + "implementations compliant with official GA4GH specifications. For services with " + "custom APIs not standardized by GA4GH, or implementations diverging from official " + "GA4GH specifications, use a different namespace (e.g. your organization's reverse " + "domain name).", + example="org.ga4gh", + ) + artifact: str = Field( + ..., + description="Name of the API or GA4GH specification implemented. Official GA4GH " + "types should be assigned as part of standards approval process. Custom artifacts " + "are supported.", + example="beacon", + ) + version: str = Field( + ..., + description="Version of the API or specification. GA4GH specifications use " + "semantic versioning.", + example="1.0.0", + ) + + +class Organization(BaseModel): + """Organization responsible for a GA4GH service.""" + + name: str = Field( + ..., + description="Name of the organization responsible for the service", + example="My organization", + ) + url: AnyUrl = Field( + ..., + description="URL of the website of the organization (RFC 3986 format)", + example="https://example.com", + ) + + +class Service(BaseModel): + """GA4GH service.""" + + id: str = Field( + ..., + description="Unique ID of this service. Reverse domain name notation is " + "recommended, though not required. The identifier should attempt to be globally " + "unique so it can be used in downstream aggregator services e.g. Service Registry.", + example="org.ga4gh.myservice", + ) + name: str = Field( + ..., + description="Name of this service. Should be human readable.", + example="My project", + ) + type: ServiceType + description: Optional[str] = Field( + default=None, + description="Description of the service. Should be human readable and provide " + "information about the service.", + example="This service provides...", + ) + organization: Organization = Field( + ..., description="Organization providing the service" + ) + contactUrl: Optional[AnyUrl] = Field( + default=None, + description="URL of the contact for the provider of this service, e.g. a link to a " + "contact form (RFC 3986 format), or an email (RFC 2368 format).", + example="mailto:support@example.com", + ) + documentationUrl: Optional[AnyUrl] = Field( + default=None, + description="URL of the documentation of this service (RFC 3986 format). This " + "should help someone learn how to use your service, including any specifics " + "required to access data, e.g. authentication.", + example="https://docs.myservice.example.com", + ) + createdAt: Optional[datetime] = Field( + default=None, + description="Timestamp describing when the service was first deployed and available" + " (RFC 3339 format)", + example="2019-06-04T12:58:19Z", + ) + updatedAt: Optional[datetime] = Field( + default=None, + description="Timestamp describing when the service was last updated (RFC 3339" + " format)", + example="2019-06-04T12:58:19Z", + ) + environment: Optional[str] = Field( + default=None, + description="Environment the service is running in. Use this to distinguish between" + " production, development and testing/staging deployments. Suggested values are " + "prod, test, dev, staging. However this is advised and not enforced.", + example="test", + ) + version: str = Field( + ..., + description="Version of the service being described. Semantic versioning is " + "recommended, but other identifiers, such as dates or commit hashes, are also " + "allowed. The version should be changed whenever the service is updated.", + example="1.0.0", ) class TesState(Enum): - """Enum for TES task states.""" + """Task state.""" UNKNOWN = "UNKNOWN" QUEUED = "QUEUED" @@ -320,93 +403,161 @@ class TesState(Enum): RUNNING = "RUNNING" PAUSED = "PAUSED" COMPLETE = "COMPLETE" - CANCELED = "CANCELED" EXECUTOR_ERROR = "EXECUTOR_ERROR" SYSTEM_ERROR = "SYSTEM_ERROR" + CANCELED = "CANCELED" + PREEMPTED = "PREEMPTED" + CANCELING = "CANCELING" -class TesTask(BaseModel): - """Represents a TES task. - - Attributes: - id (Optional[str]): Task identifier. - state (Optional[TesState]): State of the task. - name (Optional[str]): Name of the task. - description (Optional[str]): Description of the task. - executors (List[TesExecutor]): Executors for the task. - inputs (Optional[List[TesInput]]): Input files. - outputs (Optional[List[TesOutput]]): Output files. - resources (Optional[TesResources]): Resource requirements. - creation_time (Optional[str]): Creation time. - start_time (Optional[str]): Start time. - end_time (Optional[str]): End time. - logs (Optional[List[TesLogs]]): Logs for the task. - """ +class TesTaskLog(BaseModel): + """TaskLog describes logging information related to a Task.""" - id: Optional[str] = Field( - None, - description="Task identifier, assigned by the server.", - ) - state: Optional[TesState] = Field( - TesState.UNKNOWN, - description="Task state.", + logs: List[TesExecutorLog] = Field(..., description="Logs for each executor") + metadata: Optional[Dict[str, str]] = Field( + default=None, + description="Arbitrary logging metadata included by the implementation.", + example={"host": "worker-001", "slurmm_id": 123456}, ) - name: Optional[str] = Field( - None, - description="Optional, must be a non-empty string.", - example="Example Task", + start_time: Optional[str] = Field( + default=None, + description="When the task started, in RFC 3339 format.", + example="2020-10-02T10:00:00-05:00", ) - description: Optional[str] = Field( - None, - description="Optional, must be a non-empty string.", + end_time: Optional[str] = Field( + default=None, + description="When the task ended, in RFC 3339 format.", + example="2020-10-02T11:00:00-05:00", ) - executors: List[TesExecutor] = Field( + outputs: List[TesOutputFileLog] = Field( ..., - description="At least one Executor must be present.", + description="Information about all output files. Directory outputs are\nflattened " + "into separate items.", ) - inputs: Optional[List[TesInput]] = Field( - None, - description="Array of input files.", + system_logs: Optional[List[str]] = Field( + default=None, + description="System logs are any logs the system decides are relevant,\nwhich are " + "not tied directly to an Executor process.\nContent is implementation specific: " + "format, size, etc.\n\nSystem logs may be collected here to provide convenient " + "access.\n\nFor example, the system may include the name of the host\nwhere the " + "task is executing, an error message that caused\na SYSTEM_ERROR state (e.g. disk " + "is full), etc.\n\nSystem logs are only included in the FULL task view.", ) - outputs: Optional[List[TesOutput]] = Field( - None, - description="Array of output files.", + + +class TesServiceType(ServiceType): + """Type of a TES service.""" + + artifact: Artifact = Field(..., example="tes") # type: ignore + + +class TesServiceInfo(Service): + """ServiceInfo describes the service that is running the TES API.""" + + storage: Optional[List[str]] = Field( + default=None, + description="Lists some, but not necessarily all, storage locations supported\nby " + "the service.", + example=[ + "file:///path/to/local/funnel-storage", + "s3://ohsu-compbio-funnel/storage", + ], ) - resources: Optional[TesResources] = Field( - None, - description="Resource requirements.", + tesResources_backend_parameters: Optional[List[str]] = Field( + default=None, + description="Lists all tesResources.backend_parameters keys supported\nby the " + "service", + example=["VmSize"], ) - creation_time: Optional[str] = Field( - None, - description="Time the task was created, in RFC 3339 format.", - example="2020-10-02T10:00:00-05:00", + type: Optional[TesServiceType] = None # type: ignore + + +class TesTask(BaseModel): + """Task describes a task to be run.""" + + id: Optional[str] = Field( + default=None, + description="Task identifier assigned by the server.", + example="job-0012345", ) - start_time: Optional[str] = Field( - None, - description="Time the task was started, in RFC 3339 format.", - example="2020-10-02T10:05:00-05:00", + state: Optional[TesState] = TesState.UNKNOWN + name: Optional[str] = Field(None, description="User-provided task name.") + description: Optional[str] = Field( + default=None, + description="Optional user-provided description of task for documentation " + "purposes.", ) - end_time: Optional[str] = Field( - None, - description="Time the task ended, in RFC 3339 format.", - example="2020-10-02T11:00:00-05:00", + inputs: Optional[List[TesInput]] = Field( + default=None, + description="Input files that will be used by the task. Inputs will be downloaded" + "\nand mounted into the executor container as defined by the task " + "request\ndocument.", + example=[{"url": "s3://my-object-store/file1", "path": "/data/file1"}], ) - logs: Optional[List[TesLogs]] = Field( - None, - description="Task logs.\n\n" - "If no logs are available, it should be set to an empty array.", + outputs: Optional[List[TesOutput]] = Field( + default=None, + description="Output files.\nOutputs will be uploaded from the executor container " + "to long-term storage.", + example=[ + { + "path": "/data/outfile", + "url": "s3://my-object-store/outfile-1", + "type": "FILE", + } + ], + ) + resources: Optional[TesResources] = None + executors: List[TesExecutor] = Field( + ..., + description="An array of executors to be run. Each of the executors will run " + "one\nat a time sequentially. Each executor is a different command that\nwill be " + "run, and each can utilize a different docker image. But each of\nthe executors " + "will see the same mapped inputs and volumes that are declared\nin the parent " + "CreateTask message.\n\nExecution stops on the first error.", + ) + volumes: Optional[List[str]] = Field( + default=None, + description="Volumes are directories which may be used to share data between\n" + "Executors. Volumes are initialized as empty directories by the\nsystem when the " + "task starts and are mounted at the same path\nin each Executor.\n\nFor example, " + "given a volume defined at `/vol/A`,\nexecutor 1 may write a file to " + "`/vol/A/exec1.out.txt`, then\nexecutor 2 may read from that file.\n\n(Essentially," + "this translates to a `docker run -v` flag where\nthe container path is the same " + "for each executor).", + example=["/vol/A/"], + ) + tags: Optional[Dict[str, str]] = Field( + default=None, + description="A key-value map of arbitrary tags. These can be used to store " + '"meta-data\nand annotations about a task. Example:\n```\n{\n "tags" : {\n ' + '"WORKFLOW_ID" : "cwl-01234",\n "PROJECT_GROUP" : "alice-lab"\n }\n}\n```', + example={"WORKFLOW_ID": "cwl-01234", "PROJECT_GROUP": "alice-lab"}, + ) + logs: Optional[List[TesTaskLog]] = Field( + default=None, + description="Task logging information.\nNormally, this will contain only one entry," + " but in the case where\na task fails and is retried, an entry will be appended to " + "this list.", + ) + creation_time: Optional[str] = Field( + default=None, + description="Date + time the task was created, in RFC 3339 format.\nThis is set by" + " the system, not the client.", + example="2020-10-02T10:00:00-05:00", ) -class TesTaskLog(BaseModel): - """Represents the log for a TES task. - - Attributes: - logs (Optional[List[TesExecutorLog]]): Executor logs. - """ +class TesListTasksResponse(BaseModel): + """ListTasksResponse describes a response from the ListTasks endpoint.""" - logs: Optional[List[TesExecutorLog]] = Field( - None, - description="Executor logs.\n\n" - "If no logs are available, it should be set to an empty array.", + tasks: List[TesTask] = Field( + ..., + description="List of tasks. These tasks will be based on the original " + "submitted\ntask document, but with other fields, such as the job state " + "and\nlogging info, added/changed as the job progresses.", + ) + next_page_token: Optional[str] = Field( + default=None, + description="Token used to return the next page of results. This value can be " + "used\nin the `page_token` field of the next ListTasks request.", ) From 4564e29718ead8c60d4639cf28f0726c2f3fc157 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Sun, 30 Jun 2024 11:47:21 +0530 Subject: [PATCH 04/11] format --- tesk/api/ga4gh/tes/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tesk/api/ga4gh/tes/models.py b/tesk/api/ga4gh/tes/models.py index b4a70072..393aab74 100644 --- a/tesk/api/ga4gh/tes/models.py +++ b/tesk/api/ga4gh/tes/models.py @@ -469,7 +469,7 @@ class TesServiceInfo(Service): "service", example=["VmSize"], ) - type: Optional[TesServiceType] = None # type: ignore + type: Optional[TesServiceType] = None # type: ignore class TesTask(BaseModel): From 3f2afb5f127f35f7861cd5b34fba46b5a5b37d19 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 3 Jul 2024 09:03:56 +0530 Subject: [PATCH 05/11] indent 4 --- docs/source/conf.py | 34 +++++----- pyproject.toml | 2 +- tesk/api/ga4gh/tes/controllers.py | 66 +++++++++--------- tesk/app.py | 66 +++++++++--------- tesk/exceptions.py | 108 +++++++++++++++--------------- 5 files changed, 139 insertions(+), 137 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2761d14e..5447ee41 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- def _get_project_meta(): - _pyproject_path = Path(__file__).parents[2] / "pyproject.toml" - with open(_pyproject_path, mode="rb") as pyproject: - return tomli.load(pyproject)["tool"]["poetry"] + _pyproject_path = Path(__file__).parents[2] / "pyproject.toml" + with open(_pyproject_path, mode="rb") as pyproject: + return tomli.load(pyproject)["tool"]["poetry"] pkg_meta = _get_project_meta() @@ -35,20 +35,20 @@ def _get_project_meta(): # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.doctest", - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.viewcode", - "sphinx.ext.autosummary", - # Used to write beautiful docstrings: - "sphinx.ext.napoleon", - # Used to include .md files: - "m2r2", - # Used to insert typehints into the final docs: - "sphinx_autodoc_typehints", - # Used to embed values from the source code into the docs: - "added_value", + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.autosummary", + # Used to write beautiful docstrings: + "sphinx.ext.napoleon", + # Used to include .md files: + "m2r2", + # Used to insert typehints into the final docs: + "sphinx_autodoc_typehints", + # Used to embed values from the source code into the docs: + "added_value", ] # Set `typing.TYPE_CHECKING` to `True`: diff --git a/pyproject.toml b/pyproject.toml index 65ea0ed9..1e09c08e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ exclude = [ "tesk/services/*", "tests/test_unit/test_services/*", ] -indent-width = 2 +indent-width = 4 [tool.ruff.format] docstring-code-format = true diff --git a/tesk/api/ga4gh/tes/controllers.py b/tesk/api/ga4gh/tes/controllers.py index 53d6b2c3..c92497fe 100644 --- a/tesk/api/ga4gh/tes/controllers.py +++ b/tesk/api/ga4gh/tes/controllers.py @@ -12,60 +12,60 @@ # POST /tasks/{id}:cancel @log_traffic def CancelTask(id, *args, **kwargs) -> dict: # type: ignore - """Cancel unfinished task. + """Cancel unfinished task. - Args: - id: Task identifier. - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + id: Task identifier. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # POST /tasks @log_traffic def CreateTask(*args, **kwargs) -> dict: # type: ignore - """Create task. + """Create task. - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # GET /tasks/service-info @log_traffic def GetServiceInfo(*args, **kwargs) -> dict: # type: ignore - """Get service info. + """Get service info. - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # GET /tasks @log_traffic def ListTasks(*args, **kwargs) -> dict: # type: ignore - """List all available tasks. + """List all available tasks. - Args: - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass # GET /tasks @log_traffic def GetTask(*args, **kwargs) -> dict: # type: ignore - """Get info for individual task. - - Args: - id: Task identifier. - *args: Variable length argument list. - **kwargs: Arbitrary keyword arguments. - """ - pass + """Get info for individual task. + + Args: + id: Task identifier. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + pass diff --git a/tesk/app.py b/tesk/app.py index 6ace9684..3927d5bf 100644 --- a/tesk/app.py +++ b/tesk/app.py @@ -11,41 +11,43 @@ def init_app() -> FlaskApp: - """Initialize and return the FOCA app. - - This function initializes the FOCA app by loading the configuration - from the environment variable `TESK_FOCA_CONFIG_PATH` if set, or from - the default path if not. It raises a `FileNotFoundError` if the - configuration file is not found. - - Returns: - FlaskApp: A Connexion application instance. - - Raises: - FileNotFoundError: If the configuration file is not found. - """ - # Determine the configuration path - config_path_env = os.getenv("TESK_FOCA_CONFIG_PATH") - if config_path_env: - config_path = Path(config_path_env).resolve() - else: - config_path = (Path(__file__).parents[1] / "deployment" / "config.yaml").resolve() - - # Check if the configuration file exists - if not config_path.exists(): - raise FileNotFoundError(f"Config file not found at: {config_path}") - - foca = Foca( - config_file=config_path, - ) - return foca.create_app() + """Initialize and return the FOCA app. + + This function initializes the FOCA app by loading the configuration + from the environment variable `TESK_FOCA_CONFIG_PATH` if set, or from + the default path if not. It raises a `FileNotFoundError` if the + configuration file is not found. + + Returns: + FlaskApp: A Connexion application instance. + + Raises: + FileNotFoundError: If the configuration file is not found. + """ + # Determine the configuration path + config_path_env = os.getenv("TESK_FOCA_CONFIG_PATH") + if config_path_env: + config_path = Path(config_path_env).resolve() + else: + config_path = ( + Path(__file__).parents[1] / "deployment" / "config.yaml" + ).resolve() + + # Check if the configuration file exists + if not config_path.exists(): + raise FileNotFoundError(f"Config file not found at: {config_path}") + + foca = Foca( + config_file=config_path, + ) + return foca.create_app() def main() -> None: - """Run FOCA application.""" - app = init_app() - app.run(port=app.port) + """Run FOCA application.""" + app = init_app() + app.run(port=app.port) if __name__ == "__main__": - main() + main() diff --git a/tesk/exceptions.py b/tesk/exceptions.py index ffe58fd5..8d00301c 100644 --- a/tesk/exceptions.py +++ b/tesk/exceptions.py @@ -1,68 +1,68 @@ """App exceptions.""" from connexion.exceptions import ( - BadRequestProblem, - ExtraParameterProblem, - Forbidden, - OAuthProblem, - Unauthorized, + BadRequestProblem, + ExtraParameterProblem, + Forbidden, + OAuthProblem, + Unauthorized, ) from werkzeug.exceptions import ( - BadRequest, - InternalServerError, - NotFound, + BadRequest, + InternalServerError, + NotFound, ) # exceptions raised in app context exceptions = { - Exception: { - "title": "Unknown error", - "detail": "An unexpected error occurred.", - "status": 500, - }, - BadRequestProblem: { - "title": "Bad request", - "detail": "The request is malformed", - "status": 400, - }, - BadRequest: { - "title": "Bad request", - "detail": "The request is malformed.", - "status": 400, - }, - ExtraParameterProblem: { - "title": "Bad request", - "detail": "The request is malformed.", - "status": 400, - }, - Unauthorized: { - "title": "Unauthorized", - "detail": "The request is unauthorized.", - "status": 401, - }, - OAuthProblem: { - "title": "Unauthorized", - "detail": "The request is unauthorized.", - "status": 401, - }, - Forbidden: { - "title": "Forbidden", - "detail": "The requester is not authorized to perform this action.", - "status": 403, - }, - NotFound: { - "title": "Not found", - "detail": "The requested resource wasn't found.", - "status": 404, - }, - InternalServerError: { - "title": "Internal server error", - "detail": "An unexpected error occurred.", - "status": 500, - }, + Exception: { + "title": "Unknown error", + "detail": "An unexpected error occurred.", + "status": 500, + }, + BadRequestProblem: { + "title": "Bad request", + "detail": "The request is malformed", + "status": 400, + }, + BadRequest: { + "title": "Bad request", + "detail": "The request is malformed.", + "status": 400, + }, + ExtraParameterProblem: { + "title": "Bad request", + "detail": "The request is malformed.", + "status": 400, + }, + Unauthorized: { + "title": "Unauthorized", + "detail": "The request is unauthorized.", + "status": 401, + }, + OAuthProblem: { + "title": "Unauthorized", + "detail": "The request is unauthorized.", + "status": 401, + }, + Forbidden: { + "title": "Forbidden", + "detail": "The requester is not authorized to perform this action.", + "status": 403, + }, + NotFound: { + "title": "Not found", + "detail": "The requested resource wasn't found.", + "status": 404, + }, + InternalServerError: { + "title": "Internal server error", + "detail": "An unexpected error occurred.", + "status": 500, + }, } # exceptions raised outside of app context class ValidationError(Exception): - """Value or object is not compatible with required type or schema.""" + """Value or object is not compatible with required type or schema.""" From a1c0c877d5e3434b01ea6917c103252e63083878 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 3 Jul 2024 09:06:21 +0530 Subject: [PATCH 06/11] remove return type from docstring --- tesk/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tesk/app.py b/tesk/app.py index 3927d5bf..aeeee964 100644 --- a/tesk/app.py +++ b/tesk/app.py @@ -19,7 +19,7 @@ def init_app() -> FlaskApp: configuration file is not found. Returns: - FlaskApp: A Connexion application instance. + A Connexion application instance. Raises: FileNotFoundError: If the configuration file is not found. From 7726d68b0aa6b8d098cce10cc33b700c131cfa14 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 3 Jul 2024 09:08:38 +0530 Subject: [PATCH 07/11] minor --- tesk/app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tesk/app.py b/tesk/app.py index aeeee964..f09e525c 100644 --- a/tesk/app.py +++ b/tesk/app.py @@ -25,8 +25,7 @@ def init_app() -> FlaskApp: FileNotFoundError: If the configuration file is not found. """ # Determine the configuration path - config_path_env = os.getenv("TESK_FOCA_CONFIG_PATH") - if config_path_env: + if config_path_env := os.getenv("TESK_FOCA_CONFIG_PATH"): config_path = Path(config_path_env).resolve() else: config_path = ( From ffc01f6dc845cd314e7d190b545ce127b89436bb Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 3 Jul 2024 09:26:09 +0530 Subject: [PATCH 08/11] change indent to 4 --- tesk/api/ga4gh/tes/models.py | 987 ++++++++++++++++++----------------- 1 file changed, 497 insertions(+), 490 deletions(-) diff --git a/tesk/api/ga4gh/tes/models.py b/tesk/api/ga4gh/tes/models.py index 393aab74..f64f1222 100644 --- a/tesk/api/ga4gh/tes/models.py +++ b/tesk/api/ga4gh/tes/models.py @@ -8,556 +8,563 @@ class TesCancelTaskResponse(BaseModel): - """CancelTaskResponse describes a response from the CancelTask endpoint.""" + """CancelTaskResponse describes a response from the CancelTask endpoint.""" - pass + pass class TesCreateTaskResponse(BaseModel): - """CreateTaskResponse describes a response from the CreateTask endpoint. + """CreateTaskResponse describes a response from the CreateTask endpoint. - It will include the task ID that can be used to look up the status of the job. - """ + It will include the task ID that can be used to look up the status of the job. + """ - id: str = Field(..., description="Task identifier assigned by the server.") + id: str = Field(..., description="Task identifier assigned by the server.") class TesExecutor(BaseModel): - """An executor is a command to be run in a container.""" - - image: str = Field( - ..., - description=( - "Name of the container image. The string will be passed as the image" - "\nargument to the containerization run command. Examples:\n - `ubuntu`\n - " - "`quay.io/aptible/ubuntu`\n - `gcr.io/my-org/my-image`\n - " - "`myregistryhost:5000/fedora/httpd:version1.0`" - ), - example="ubuntu:20.04", - ) - command: List[str] = Field( - ..., - description="A sequence of program arguments to execute, where the first argument\n" - 'is the program to execute (i.e. argv). Example:\n```\n{\n "command" : ["/bin/md5"' - ', "/data/file1"]\n}\n```', - example=["/bin/md5", "/data/file1"], - ) - workdir: Optional[str] = Field( - default=None, - description=( - "The working directory that the command will be executed in.\nIf not " - "defined, the system will default to the directory set by\nthe container image." - ), - example="/data/", - ) - stdin: Optional[str] = Field( - default=None, - description=( - "Path inside the container to a file which will be piped\nto the " - "executor's stdin. This must be an absolute path. This mechanism" - "\ncould be used in conjunction with the input declaration to " - "process\na data file using a tool that expects STDIN.\n\nFor " - "example, to get the MD5 sum of a file by reading it into the " - 'STDIN\n```\n{\n "command" : ["/bin/md5"],\n "stdin" : "/data/file1"' - "\n}\n```" - ), - example="/data/file1", - ) - stdout: Optional[str] = Field( - default=None, - description="Path inside the container to a file where the executor's\nstdout will " - 'be written to. Must be an absolute path. Example:\n```\n{\n "stdout" : ' - '"/tmp/stdout.log"\n}\n```', - example="/tmp/stdout.log", - ) - stderr: Optional[str] = Field( - default=None, - description="Path inside the container to a file where the executor's\nstderr " - 'will be written to. Must be an absolute path. Example:\n```\n{\n "stderr" : ' - '"/tmp/stderr.log"\n}\n```', - example="/tmp/stderr.log", - ) - env: Optional[Dict[str, str]] = Field( - default=None, - description="Environmental variables to set within the container. Example:\n```\n" - '{\n "env" : {\n "ENV_CONFIG_PATH" : "/data/config.file",\n "BLASTDB" : ' - '"/data/GRC38",\n "HMMERDB" : "/data/hmmer"\n }\n}\n```', - example={"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}, - ) - ignore_error: Optional[bool] = Field( - default=None, - description="Default behavior of running an array of executors is that " - "execution\nstops on the first error. If `ignore_error` is `True`, then " - "the\nrunner will record error exit codes, but will continue on to the " - "next\ntesExecutor.", - ) + """An executor is a command to be run in a container.""" + + image: str = Field( + ..., + description=( + "Name of the container image. The string will be passed as the image" + "\nargument to the containerization run command. Examples:\n - `ubuntu`" + "\n - `quay.io/aptible/ubuntu`\n - `gcr.io/my-org/my-image`\n - " + "`myregistryhost:5000/fedora/httpd:version1.0`" + ), + example="ubuntu:20.04", + ) + command: List[str] = Field( + ..., + description="A sequence of program arguments to execute, where the first" + "argument\n is the program to execute (i.e. argv). Example:\n```\n{\n " + '"command" : ["/bin/md5" , "/data/file1"]\n}\n```', + example=["/bin/md5", "/data/file1"], + ) + workdir: Optional[str] = Field( + default=None, + description=( + "The working directory that the command will be executed in.\nIf not " + "defined, the system will default to the directory set by\nthe container " + "image." + ), + example="/data/", + ) + stdin: Optional[str] = Field( + default=None, + description=( + "Path inside the container to a file which will be piped\nto the " + "executor's stdin. This must be an absolute path. This mechanism" + "\ncould be used in conjunction with the input declaration to " + "process\na data file using a tool that expects STDIN.\n\nFor " + "example, to get the MD5 sum of a file by reading it into the " + 'STDIN\n```\n{\n "command" : ["/bin/md5"],\n "stdin" : "/data/file1"' + "\n}\n```" + ), + example="/data/file1", + ) + stdout: Optional[str] = Field( + default=None, + description="Path inside the container to a file where the executor's\nstdout " + 'will be written to. Must be an absolute path. Example:\n```\n{\n "stdout" : ' + '"/tmp/stdout.log"\n}\n```', + example="/tmp/stdout.log", + ) + stderr: Optional[str] = Field( + default=None, + description="Path inside the container to a file where the executor's\nstderr " + 'will be written to. Must be an absolute path. Example:\n```\n{\n "stderr" : ' + '"/tmp/stderr.log"\n}\n```', + example="/tmp/stderr.log", + ) + env: Optional[Dict[str, str]] = Field( + default=None, + description="Environmental variables to set within the container. " + 'Example:\n```\n {\n "env" : {\n "ENV_CONFIG_PATH" : "/data/config.file",' + '\n "BLASTDB" : "/data/GRC38",\n "HMMERDB" : "/data/hmmer"\n }\n}\n```', + example={"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}, + ) + ignore_error: Optional[bool] = Field( + default=None, + description="Default behavior of running an array of executors is that " + "execution\nstops on the first error. If `ignore_error` is `True`, then " + "the\nrunner will record error exit codes, but will continue on to the " + "next\ntesExecutor.", + ) class TesExecutorLog(BaseModel): - """ExecutorLog describes logging information related to an Executor.""" - - start_time: Optional[str] = Field( - default=None, - description="Time the executor started, in RFC 3339 format.", - example="2020-10-02T10:00:00-05:00", - ) - end_time: Optional[str] = Field( - default=None, - description="Time the executor ended, in RFC 3339 format.", - example="2020-10-02T11:00:00-05:00", - ) - stdout: Optional[str] = Field( - default=None, - description="Stdout content.\n\nThis is meant for convenience. No guarantees are " - "made about the content.\nImplementations may chose different approaches: only the " - "head, only the tail,\na URL reference only, etc.\n\nIn order to capture the full " - "stdout client should set Executor.stdout\nto a container file path, and use Task." - "outputs to upload that file\nto permanent storage.", - ) - stderr: Optional[str] = Field( - default=None, - description="Stderr content.\n\nThis is meant for convenience. No guarantees are " - "made about the content.\nImplementations may chose different approaches: only the " - "head, only the tail,\na URL reference only, etc.\n\nIn order to capture the full " - "stderr client should set Executor.stderr\nto a container file path, and use Task." - "outputs to upload that file\nto permanent storage.", - ) - exit_code: int = Field(..., description="Exit code.") + """ExecutorLog describes logging information related to an Executor.""" + + start_time: Optional[str] = Field( + default=None, + description="Time the executor started, in RFC 3339 format.", + example="2020-10-02T10:00:00-05:00", + ) + end_time: Optional[str] = Field( + default=None, + description="Time the executor ended, in RFC 3339 format.", + example="2020-10-02T11:00:00-05:00", + ) + stdout: Optional[str] = Field( + default=None, + description="Stdout content.\n\nThis is meant for convenience. No guarantees " + "are made about the content.\nImplementations may chose different approaches: " + "only the head, only the tail,\na URL reference only, etc.\n\nIn order to " + "capture the full stdout client should set Executor.stdout\nto a container " + "file path, and use Task. outputs to upload that file\nto permanent storage.", + ) + stderr: Optional[str] = Field( + default=None, + description="Stderr content.\n\nThis is meant for convenience. No guarantees " + "are made about the content.\nImplementations may chose different approaches: " + "only the head, only the tail,\na URL reference only, etc.\n\nIn order to " + "capture the full stderr client should set Executor.stderr\nto a container file" + " path, and use Task. outputs to upload that file\nto permanent storage.", + ) + exit_code: int = Field(..., description="Exit code.") class TesFileType(Enum): - """Define if input/output element is a file or a directory. + """Define if input/output element is a file or a directory. - It is not required that the user provide this value, but it is required that - the server fill in the value once the information is available at run time. - """ + It is not required that the user provide this value, but it is required that + the server fill in the value once the information is available at run time. + """ - FILE = "FILE" - DIRECTORY = "DIRECTORY" + FILE = "FILE" + DIRECTORY = "DIRECTORY" class TesInput(BaseModel): - """Input describes Task input files.""" - - name: Optional[str] = None - description: Optional[str] = None - url: Optional[str] = Field( - default=None, - description='REQUIRED, unless "content" is set.\n\nURL in long term storage, for ' - "example:\n - s3://my-object-store/file1\n - gs://my-bucket/file2\n - " - "file:///path/to/my/file\n - /path/to/my/file", - example="s3://my-object-store/file1", - ) - path: str = Field( - ..., - description="Path of the file inside the container.\nMust be an absolute path.", - example="/data/file1", - ) - type: Optional[TesFileType] = TesFileType.FILE - content: Optional[str] = Field( - default=None, - description="File content literal.\n\nImplementations should support a minimum of " - "128 KiB in this field\nand may define their own maximum.\n\nUTF-8 encoded\n\nIf " - 'content is not empty, "url" must be ignored.', - ) - streamable: Optional[bool] = Field( - default=None, - description="Indicate that a file resource could be accessed using a streaming" - "\ninterface, ie a FUSE mounted s3 object. This flag indicates that\nusing a " - "streaming mount, as opposed to downloading the whole file to\nthe local scratch " - "space, may be faster despite the latency and\noverhead. This does not mean that " - "the backend will use a streaming\ninterface, as it may not be provided by the " - "vendor, but if the\ncapacity is available it can be used without degrading " - "the\nperformance of the underlying program.", - ) + """Input describes Task input files.""" + + name: Optional[str] = None + description: Optional[str] = None + url: Optional[str] = Field( + default=None, + description='REQUIRED, unless "content" is set.\n\nURL in long term storage, ' + "for example:\n - s3://my-object-store/file1\n - gs://my-bucket/file2\n - " + "file:///path/to/my/file\n - /path/to/my/file", + example="s3://my-object-store/file1", + ) + path: str = Field( + ..., + description="Path of the file inside the container.\nMust be an absolute path.", + example="/data/file1", + ) + type: Optional[TesFileType] = TesFileType.FILE + content: Optional[str] = Field( + default=None, + description="File content literal.\n\nImplementations should support a minimum " + "of 128 KiB in this field\nand may define their own maximum.\n\nUTF-8 encoded" + '\n\nIf content is not empty, "url" must be ignored.', + ) + streamable: Optional[bool] = Field( + default=None, + description="Indicate that a file resource could be accessed using a streaming" + "\ninterface, ie a FUSE mounted s3 object. This flag indicates that\nusing a " + "streaming mount, as opposed to downloading the whole file to\nthe local " + "scratch space, may be faster despite the latency and\noverhead. This does not " + "mean that the backend will use a streaming\ninterface, as it may not be " + "provided by the vendor, but if the\ncapacity is available it can be used " + "without degrading the\nperformance of the underlying program.", + ) class TesOutput(BaseModel): - """Output describes Task output files.""" - - name: Optional[str] = Field(None, description="User-provided name of output file") - description: Optional[str] = Field( - default=None, - description="Optional users provided description field, can be used for " - "documentation.", - ) - url: str = Field( - ..., - description="URL at which the TES server makes the output accessible after the " - "task is complete.\nWhen tesOutput.path contains wildcards, it must be a " - "directory; see\n`tesOutput.path_prefix` for details on how output URLs are " - "constructed in this case.\nFor Example:\n - `s3://my-object-store/file1`\n - " - "`gs://my-bucket/file2`\n - `file:///path/to/my/file`", - ) - path: str = Field( - ..., - description="Absolute path of the file inside the container.\nMay contain pattern " - "matching wildcards to select multiple outputs at once, but mind\nimplications for " - "`tesOutput.url` and `tesOutput.path_prefix`.\nOnly wildcards defined in IEEE Std " - "1003.1-2017 (POSIX), 12.3 are supported; " - "see\nhttps://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13", - ) - path_prefix: Optional[str] = Field( - default=None, - description="Prefix to be removed from matching outputs if `tesOutput.path` " - "contains wildcards;\noutput URLs are constructed by appending pruned paths to the " - "directory specified\nin `tesOutput.url`.\nRequired if `tesOutput.path` contains " - "wildcards, ignored otherwise.", - ) - type: Optional[TesFileType] = TesFileType.FILE + """Output describes Task output files.""" + + name: Optional[str] = Field(None, description="User-provided name of output file") + description: Optional[str] = Field( + default=None, + description="Optional users provided description field, can be used for " + "documentation.", + ) + url: str = Field( + ..., + description="URL at which the TES server makes the output accessible after the " + "task is complete.\nWhen tesOutput.path contains wildcards, it must be a " + "directory; see\n`tesOutput.path_prefix` for details on how output URLs are " + "constructed in this case.\nFor Example:\n - `s3://my-object-store/file1`\n - " + "`gs://my-bucket/file2`\n - `file:///path/to/my/file`", + ) + path: str = Field( + ..., + description="Absolute path of the file inside the container.\nMay contain " + "pattern matching wildcards to select multiple outputs at once, but mind\n" + "implications for `tesOutput.url` and `tesOutput.path_prefix`.\nOnly wildcards " + "defined in IEEE Std 1003.1-2017 (POSIX), 12.3 are supported; " + "see\nhttps://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13", + ) + path_prefix: Optional[str] = Field( + default=None, + description="Prefix to be removed from matching outputs if `tesOutput.path` " + "contains wildcards;\noutput URLs are constructed by appending pruned paths to " + "the directory specified\nin `tesOutput.url`.\nRequired if `tesOutput.path` " + "contains wildcards, ignored otherwise.", + ) + type: Optional[TesFileType] = TesFileType.FILE class TesOutputFileLog(BaseModel): - """OutputFileLog describes a single output file. - - This describes file details after the task has completed successfully, for logging - purposes. - """ - - url: str = Field( - ..., description="URL of the file in storage, e.g. s3://bucket/file.txt" - ) - path: str = Field( - ..., - description="Path of the file inside the container. Must be an absolute path.", - ) - size_bytes: str = Field( - ..., - description="Size of the file in bytes. Note, this is currently coded as a " - "string\nbecause official JSON doesn't support int64 numbers.", - example=["1024"], - ) + """OutputFileLog describes a single output file. + + This describes file details after the task has completed successfully, for logging + purposes. + """ + + url: str = Field( + ..., description="URL of the file in storage, e.g. s3://bucket/file.txt" + ) + path: str = Field( + ..., + description="Path of the file inside the container. Must be an absolute path.", + ) + size_bytes: str = Field( + ..., + description="Size of the file in bytes. Note, this is currently coded as a " + "string\nbecause official JSON doesn't support int64 numbers.", + example=["1024"], + ) class TesResources(BaseModel): - """Resources describes the resources requested by a task.""" - - cpu_cores: Optional[int] = Field( - default=None, description="Requested number of CPUs", example=4 - ) - preemptible: Optional[bool] = Field( - default=None, - description="Define if the task is allowed to run on preemptible compute instances," - "\nfor example, AWS Spot. This option may have no effect when utilized\non some " - "backends that don't have the concept of preemptible jobs.", - example=False, - ) - ram_gb: Optional[float] = Field( - default=None, description="Requested RAM required in gigabytes (GB)", example=8 - ) - disk_gb: Optional[float] = Field( - default=None, description="Requested disk size in gigabytes (GB)", example=40 - ) - zones: Optional[List[str]] = Field( - default=None, - description="Request that the task be run in these compute zones. How this string\n" - "is utilized will be dependent on the backend system. For example, a\nsystem based " - "on a cluster queueing system may use this string to define\npriority queue to " - "which the job is assigned.", - example="us-west-1", - ) - backend_parameters: Optional[Dict[str, str]] = Field( - default=None, - description="Key/value pairs for backend configuration.\nServiceInfo shall return " - "a list of keys that a backend supports.\nKeys are case insensitive.\nIt is " - "expected that clients pass all runtime or hardware requirement key/values\nthat " - "are not mapped to existing tesResources properties to backend_parameters.\n" - "Backends shall log system warnings if a key is passed that is unsupported.\n" - "Backends shall not store or return unsupported keys if included in a task.\nIf " - "backend_parameters_strict equals true,\nbackends should fail the task if any " - "key/values are unsupported, otherwise,\nbackends should attempt to run the " - "task\nIntended uses include VM size selection, coprocessor configuration, etc." - '\nExample:\n```\n{\n "backend_parameters" : {\n "VmSize" : "Standard_D64_v3"\n' - " }\n}\n```", - example={"VmSize": "Standard_D64_v3"}, - ) - backend_parameters_strict: Optional[bool] = Field( - False, - description="If set to true, backends should fail the task if any " - "backend_parameters\nkey/values are unsupported, otherwise, backends should " - "attempt to run the task", - example=False, - ) + """Resources describes the resources requested by a task.""" + + cpu_cores: Optional[int] = Field( + default=None, description="Requested number of CPUs", example=4 + ) + preemptible: Optional[bool] = Field( + default=None, + description="Define if the task is allowed to run on preemptible compute " + "instances,\nfor example, AWS Spot. This option may have no effect when " + "utilized\non some backends that don't have the concept of preemptible jobs.", + example=False, + ) + ram_gb: Optional[float] = Field( + default=None, description="Requested RAM required in gigabytes (GB)", example=8 + ) + disk_gb: Optional[float] = Field( + default=None, description="Requested disk size in gigabytes (GB)", example=40 + ) + zones: Optional[List[str]] = Field( + default=None, + description="Request that the task be run in these compute zones. How this " + "string\nis utilized will be dependent on the backend system. For example, " + "a\nsystem based on a cluster queueing system may use this string to define\n" + "priority queue to which the job is assigned.", + example="us-west-1", + ) + backend_parameters: Optional[Dict[str, str]] = Field( + default=None, + description="Key/value pairs for backend configuration.\nServiceInfo shall " + "return a list of keys that a backend supports.\nKeys are case insensitive.\nIt" + " is expected that clients pass all runtime or hardware requirement key/values" + "\nthat are not mapped to existing tesResources properties to " + "backend_parameters.\nBackends shall log system warnings if a key is passed " + "that is unsupported.\nBackends shall not store or return unsupported keys if " + "included in a task.\nIf backend_parameters_strict equals true,\nbackends " + "should fail the task if any key/values are unsupported, otherwise,\nbackends" + " should attempt to run the task\nIntended uses include VM size selection, " + 'coprocessor configuration, etc.\nExample:\n```\n{\n "backend_parameters" : ' + 'Example:\n```\n{\n "backend_parameters": {\n "VmSize": "Standard_D64_v3"\n' + ' }\n}\n"' + "```", + example={"VmSize": "Standard_D64_v3"}, + ) + backend_parameters_strict: Optional[bool] = Field( + False, + description="If set to true, backends should fail the task if any " + "backend_parameters\nkey/values are unsupported, otherwise, backends should " + "attempt to run the task", + example=False, + ) class Artifact(Enum): - """Artifact type.""" + """Artifact type.""" - tes = "tes" + tes = "tes" class ServiceType(BaseModel): - """Type of a GA4GH service.""" - - group: str = Field( - ..., - description="Namespace in reverse domain name format. Use `org.ga4gh` for " - "implementations compliant with official GA4GH specifications. For services with " - "custom APIs not standardized by GA4GH, or implementations diverging from official " - "GA4GH specifications, use a different namespace (e.g. your organization's reverse " - "domain name).", - example="org.ga4gh", - ) - artifact: str = Field( - ..., - description="Name of the API or GA4GH specification implemented. Official GA4GH " - "types should be assigned as part of standards approval process. Custom artifacts " - "are supported.", - example="beacon", - ) - version: str = Field( - ..., - description="Version of the API or specification. GA4GH specifications use " - "semantic versioning.", - example="1.0.0", - ) + """Type of a GA4GH service.""" + + group: str = Field( + ..., + description="Namespace in reverse domain name format. Use `org.ga4gh` for " + "implementations compliant with official GA4GH specifications. For services " + "with custom APIs not standardized by GA4GH, or implementations diverging from " + "official GA4GH specifications, use a different namespace (e.g. your " + "organization's reverse domain name).", + example="org.ga4gh", + ) + artifact: str = Field( + ..., + description="Name of the API or GA4GH specification implemented. Official GA4GH" + " types should be assigned as part of standards approval process. Custom " + "artifacts are supported.", + example="beacon", + ) + version: str = Field( + ..., + description="Version of the API or specification. GA4GH specifications use " + "semantic versioning.", + example="1.0.0", + ) class Organization(BaseModel): - """Organization responsible for a GA4GH service.""" + """Organization responsible for a GA4GH service.""" - name: str = Field( - ..., - description="Name of the organization responsible for the service", - example="My organization", - ) - url: AnyUrl = Field( - ..., - description="URL of the website of the organization (RFC 3986 format)", - example="https://example.com", - ) + name: str = Field( + ..., + description="Name of the organization responsible for the service", + example="My organization", + ) + url: AnyUrl = Field( + ..., + description="URL of the website of the organization (RFC 3986 format)", + example="https://example.com", + ) class Service(BaseModel): - """GA4GH service.""" - - id: str = Field( - ..., - description="Unique ID of this service. Reverse domain name notation is " - "recommended, though not required. The identifier should attempt to be globally " - "unique so it can be used in downstream aggregator services e.g. Service Registry.", - example="org.ga4gh.myservice", - ) - name: str = Field( - ..., - description="Name of this service. Should be human readable.", - example="My project", - ) - type: ServiceType - description: Optional[str] = Field( - default=None, - description="Description of the service. Should be human readable and provide " - "information about the service.", - example="This service provides...", - ) - organization: Organization = Field( - ..., description="Organization providing the service" - ) - contactUrl: Optional[AnyUrl] = Field( - default=None, - description="URL of the contact for the provider of this service, e.g. a link to a " - "contact form (RFC 3986 format), or an email (RFC 2368 format).", - example="mailto:support@example.com", - ) - documentationUrl: Optional[AnyUrl] = Field( - default=None, - description="URL of the documentation of this service (RFC 3986 format). This " - "should help someone learn how to use your service, including any specifics " - "required to access data, e.g. authentication.", - example="https://docs.myservice.example.com", - ) - createdAt: Optional[datetime] = Field( - default=None, - description="Timestamp describing when the service was first deployed and available" - " (RFC 3339 format)", - example="2019-06-04T12:58:19Z", - ) - updatedAt: Optional[datetime] = Field( - default=None, - description="Timestamp describing when the service was last updated (RFC 3339" - " format)", - example="2019-06-04T12:58:19Z", - ) - environment: Optional[str] = Field( - default=None, - description="Environment the service is running in. Use this to distinguish between" - " production, development and testing/staging deployments. Suggested values are " - "prod, test, dev, staging. However this is advised and not enforced.", - example="test", - ) - version: str = Field( - ..., - description="Version of the service being described. Semantic versioning is " - "recommended, but other identifiers, such as dates or commit hashes, are also " - "allowed. The version should be changed whenever the service is updated.", - example="1.0.0", - ) + """GA4GH service.""" + + id: str = Field( + ..., + description="Unique ID of this service. Reverse domain name notation is " + "recommended, though not required. The identifier should attempt to be globally" + " unique so it can be used in downstream aggregator services e.g. Service " + "Registry.", + example="org.ga4gh.myservice", + ) + name: str = Field( + ..., + description="Name of this service. Should be human readable.", + example="My project", + ) + type: ServiceType + description: Optional[str] = Field( + default=None, + description="Description of the service. Should be human readable and provide " + "information about the service.", + example="This service provides...", + ) + organization: Organization = Field( + ..., description="Organization providing the service" + ) + contactUrl: Optional[AnyUrl] = Field( + default=None, + description="URL of the contact for the provider of this service, e.g. a link " + "to a contact form (RFC 3986 format), or an email (RFC 2368 format).", + example="mailto:support@example.com", + ) + documentationUrl: Optional[AnyUrl] = Field( + default=None, + description="URL of the documentation of this service (RFC 3986 format). This " + "should help someone learn how to use your service, including any specifics " + "required to access data, e.g. authentication.", + example="https://docs.myservice.example.com", + ) + createdAt: Optional[datetime] = Field( + default=None, + description="Timestamp describing when the service was first deployed and " + "available (RFC 3339 format)", + example="2019-06-04T12:58:19Z", + ) + updatedAt: Optional[datetime] = Field( + default=None, + description="Timestamp describing when the service was last updated (RFC 3339" + " format)", + example="2019-06-04T12:58:19Z", + ) + environment: Optional[str] = Field( + default=None, + description="Environment the service is running in. Use this to distinguish " + "between production, development and testing/staging deployments. Suggested " + "values are prod, test, dev, staging. However this is advised and not " + "enforced.", + example="test", + ) + version: str = Field( + ..., + description="Version of the service being described. Semantic versioning is " + "recommended, but other identifiers, such as dates or commit hashes, are also " + "allowed. The version should be changed whenever the service is updated.", + example="1.0.0", + ) class TesState(Enum): - """Task state.""" + """Task state.""" - UNKNOWN = "UNKNOWN" - QUEUED = "QUEUED" - INITIALIZING = "INITIALIZING" - RUNNING = "RUNNING" - PAUSED = "PAUSED" - COMPLETE = "COMPLETE" - EXECUTOR_ERROR = "EXECUTOR_ERROR" - SYSTEM_ERROR = "SYSTEM_ERROR" - CANCELED = "CANCELED" - PREEMPTED = "PREEMPTED" - CANCELING = "CANCELING" + UNKNOWN = "UNKNOWN" + QUEUED = "QUEUED" + INITIALIZING = "INITIALIZING" + RUNNING = "RUNNING" + PAUSED = "PAUSED" + COMPLETE = "COMPLETE" + EXECUTOR_ERROR = "EXECUTOR_ERROR" + SYSTEM_ERROR = "SYSTEM_ERROR" + CANCELED = "CANCELED" + PREEMPTED = "PREEMPTED" + CANCELING = "CANCELING" class TesTaskLog(BaseModel): - """TaskLog describes logging information related to a Task.""" - - logs: List[TesExecutorLog] = Field(..., description="Logs for each executor") - metadata: Optional[Dict[str, str]] = Field( - default=None, - description="Arbitrary logging metadata included by the implementation.", - example={"host": "worker-001", "slurmm_id": 123456}, - ) - start_time: Optional[str] = Field( - default=None, - description="When the task started, in RFC 3339 format.", - example="2020-10-02T10:00:00-05:00", - ) - end_time: Optional[str] = Field( - default=None, - description="When the task ended, in RFC 3339 format.", - example="2020-10-02T11:00:00-05:00", - ) - outputs: List[TesOutputFileLog] = Field( - ..., - description="Information about all output files. Directory outputs are\nflattened " - "into separate items.", - ) - system_logs: Optional[List[str]] = Field( - default=None, - description="System logs are any logs the system decides are relevant,\nwhich are " - "not tied directly to an Executor process.\nContent is implementation specific: " - "format, size, etc.\n\nSystem logs may be collected here to provide convenient " - "access.\n\nFor example, the system may include the name of the host\nwhere the " - "task is executing, an error message that caused\na SYSTEM_ERROR state (e.g. disk " - "is full), etc.\n\nSystem logs are only included in the FULL task view.", - ) + """TaskLog describes logging information related to a Task.""" + + logs: List[TesExecutorLog] = Field(..., description="Logs for each executor") + metadata: Optional[Dict[str, str]] = Field( + default=None, + description="Arbitrary logging metadata included by the implementation.", + example={"host": "worker-001", "slurmm_id": 123456}, + ) + start_time: Optional[str] = Field( + default=None, + description="When the task started, in RFC 3339 format.", + example="2020-10-02T10:00:00-05:00", + ) + end_time: Optional[str] = Field( + default=None, + description="When the task ended, in RFC 3339 format.", + example="2020-10-02T11:00:00-05:00", + ) + outputs: List[TesOutputFileLog] = Field( + ..., + description="Information about all output files. Directory outputs are\n" + "flattened into separate items.", + ) + system_logs: Optional[List[str]] = Field( + default=None, + description="System logs are any logs the system decides are relevant,\nwhich " + "are not tied directly to an Executor process.\nContent is implementation " + "specific: format, size, etc.\n\nSystem logs may be collected here to provide " + "convenient access.\n\nFor example, the system may include the name of the " + "host\nwhere the task is executing, an error message that caused\na " + "SYSTEM_ERROR state (e.g. disk is full), etc.\n\nSystem logs are only included" + " in the FULL task view.", + ) class TesServiceType(ServiceType): - """Type of a TES service.""" + """Type of a TES service.""" - artifact: Artifact = Field(..., example="tes") # type: ignore + artifact: Artifact = Field(..., example="tes") # type: ignore class TesServiceInfo(Service): - """ServiceInfo describes the service that is running the TES API.""" - - storage: Optional[List[str]] = Field( - default=None, - description="Lists some, but not necessarily all, storage locations supported\nby " - "the service.", - example=[ - "file:///path/to/local/funnel-storage", - "s3://ohsu-compbio-funnel/storage", - ], - ) - tesResources_backend_parameters: Optional[List[str]] = Field( - default=None, - description="Lists all tesResources.backend_parameters keys supported\nby the " - "service", - example=["VmSize"], - ) - type: Optional[TesServiceType] = None # type: ignore + """ServiceInfo describes the service that is running the TES API.""" + + storage: Optional[List[str]] = Field( + default=None, + description="Lists some, but not necessarily all, storage locations supported\n" + "by the service.", + example=[ + "file:///path/to/local/funnel-storage", + "s3://ohsu-compbio-funnel/storage", + ], + ) + tesResources_backend_parameters: Optional[List[str]] = Field( + default=None, + description="Lists all tesResources.backend_parameters keys supported\nby the " + "service", + example=["VmSize"], + ) + type: Optional[TesServiceType] = None # type: ignore class TesTask(BaseModel): - """Task describes a task to be run.""" - - id: Optional[str] = Field( - default=None, - description="Task identifier assigned by the server.", - example="job-0012345", - ) - state: Optional[TesState] = TesState.UNKNOWN - name: Optional[str] = Field(None, description="User-provided task name.") - description: Optional[str] = Field( - default=None, - description="Optional user-provided description of task for documentation " - "purposes.", - ) - inputs: Optional[List[TesInput]] = Field( - default=None, - description="Input files that will be used by the task. Inputs will be downloaded" - "\nand mounted into the executor container as defined by the task " - "request\ndocument.", - example=[{"url": "s3://my-object-store/file1", "path": "/data/file1"}], - ) - outputs: Optional[List[TesOutput]] = Field( - default=None, - description="Output files.\nOutputs will be uploaded from the executor container " - "to long-term storage.", - example=[ - { - "path": "/data/outfile", - "url": "s3://my-object-store/outfile-1", - "type": "FILE", - } - ], - ) - resources: Optional[TesResources] = None - executors: List[TesExecutor] = Field( - ..., - description="An array of executors to be run. Each of the executors will run " - "one\nat a time sequentially. Each executor is a different command that\nwill be " - "run, and each can utilize a different docker image. But each of\nthe executors " - "will see the same mapped inputs and volumes that are declared\nin the parent " - "CreateTask message.\n\nExecution stops on the first error.", - ) - volumes: Optional[List[str]] = Field( - default=None, - description="Volumes are directories which may be used to share data between\n" - "Executors. Volumes are initialized as empty directories by the\nsystem when the " - "task starts and are mounted at the same path\nin each Executor.\n\nFor example, " - "given a volume defined at `/vol/A`,\nexecutor 1 may write a file to " - "`/vol/A/exec1.out.txt`, then\nexecutor 2 may read from that file.\n\n(Essentially," - "this translates to a `docker run -v` flag where\nthe container path is the same " - "for each executor).", - example=["/vol/A/"], - ) - tags: Optional[Dict[str, str]] = Field( - default=None, - description="A key-value map of arbitrary tags. These can be used to store " - '"meta-data\nand annotations about a task. Example:\n```\n{\n "tags" : {\n ' - '"WORKFLOW_ID" : "cwl-01234",\n "PROJECT_GROUP" : "alice-lab"\n }\n}\n```', - example={"WORKFLOW_ID": "cwl-01234", "PROJECT_GROUP": "alice-lab"}, - ) - logs: Optional[List[TesTaskLog]] = Field( - default=None, - description="Task logging information.\nNormally, this will contain only one entry," - " but in the case where\na task fails and is retried, an entry will be appended to " - "this list.", - ) - creation_time: Optional[str] = Field( - default=None, - description="Date + time the task was created, in RFC 3339 format.\nThis is set by" - " the system, not the client.", - example="2020-10-02T10:00:00-05:00", - ) + """Task describes a task to be run.""" + + id: Optional[str] = Field( + default=None, + description="Task identifier assigned by the server.", + example="job-0012345", + ) + state: Optional[TesState] = TesState.UNKNOWN + name: Optional[str] = Field(None, description="User-provided task name.") + description: Optional[str] = Field( + default=None, + description="Optional user-provided description of task for documentation " + "purposes.", + ) + inputs: Optional[List[TesInput]] = Field( + default=None, + description="Input files that will be used by the task. Inputs will be " + "downloaded\nand mounted into the executor container as defined by the task " + "request\ndocument.", + example=[{"url": "s3://my-object-store/file1", "path": "/data/file1"}], + ) + outputs: Optional[List[TesOutput]] = Field( + default=None, + description="Output files.\nOutputs will be uploaded from the executor " + "container to long-term storage.", + example=[ + { + "path": "/data/outfile", + "url": "s3://my-object-store/outfile-1", + "type": "FILE", + } + ], + ) + resources: Optional[TesResources] = None + executors: List[TesExecutor] = Field( + ..., + description="An array of executors to be run. Each of the executors will run " + "one\nat a time sequentially. Each executor is a different command that\nwill " + "be run, and each can utilize a different docker image. But each of\nthe " + "executors will see the same mapped inputs and volumes that are declared\nin " + "the parent CreateTask message.\n\nExecution stops on the first error.", + ) + volumes: Optional[List[str]] = Field( + default=None, + description="Volumes are directories which may be used to share data between\n" + "Executors. Volumes are initialized as empty directories by the\nsystem when " + "the task starts and are mounted at the same path\nin each Executor.\n\nFor " + "example, given a volume defined at `/vol/A`,\nexecutor 1 may write a file to " + "`/vol/A/exec1.out.txt`, then\nexecutor 2 may read from that file.\n\n" + "(Essentially,this translates to a `docker run -v` flag where\nthe container " + "path is the same for each executor).", + example=["/vol/A/"], + ) + tags: Optional[Dict[str, str]] = Field( + default=None, + description="A key-value map of arbitrary tags. These can be used to store " + '"meta-data\nand annotations about a task. Example:\n```\n{\n "tags" : {\n ' + ' "WORKFLOW_ID" : "cwl-01234",\n "PROJECT_GROUP" : "alice-lab"\n }\n}' + "\n```", + example={"WORKFLOW_ID": "cwl-01234", "PROJECT_GROUP": "alice-lab"}, + ) + logs: Optional[List[TesTaskLog]] = Field( + default=None, + description="Task logging information.\nNormally, this will contain only one " + "entry, but in the case where\na task fails and is retried, an entry will be " + "appended to this list.", + ) + creation_time: Optional[str] = Field( + default=None, + description="Date + time the task was created, in RFC 3339 format.\nThis is " + "set by the system, not the client.", + example="2020-10-02T10:00:00-05:00", + ) class TesListTasksResponse(BaseModel): - """ListTasksResponse describes a response from the ListTasks endpoint.""" - - tasks: List[TesTask] = Field( - ..., - description="List of tasks. These tasks will be based on the original " - "submitted\ntask document, but with other fields, such as the job state " - "and\nlogging info, added/changed as the job progresses.", - ) - next_page_token: Optional[str] = Field( - default=None, - description="Token used to return the next page of results. This value can be " - "used\nin the `page_token` field of the next ListTasks request.", - ) + """ListTasksResponse describes a response from the ListTasks endpoint.""" + + tasks: List[TesTask] = Field( + ..., + description="List of tasks. These tasks will be based on the original " + "submitted\ntask document, but with other fields, such as the job state " + "and\nlogging info, added/changed as the job progresses.", + ) + next_page_token: Optional[str] = Field( + default=None, + description="Token used to return the next page of results. This value can be " + "used\nin the `page_token` field of the next ListTasks request.", + ) From ec8849d0c90a6e43fbc713d076259f3e17bce9b2 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Wed, 3 Jul 2024 09:35:04 +0530 Subject: [PATCH 09/11] reviews --- tesk/api/ga4gh/tes/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tesk/api/ga4gh/tes/models.py b/tesk/api/ga4gh/tes/models.py index f64f1222..ed5172d9 100644 --- a/tesk/api/ga4gh/tes/models.py +++ b/tesk/api/ga4gh/tes/models.py @@ -86,7 +86,7 @@ class TesExecutor(BaseModel): example={"BLASTDB": "/data/GRC38", "HMMERDB": "/data/hmmer"}, ) ignore_error: Optional[bool] = Field( - default=None, + default=False, description="Default behavior of running an array of executors is that " "execution\nstops on the first error. If `ignore_error` is `True`, then " "the\nrunner will record error exit codes, but will continue on to the " @@ -154,7 +154,7 @@ class TesInput(BaseModel): description="Path of the file inside the container.\nMust be an absolute path.", example="/data/file1", ) - type: Optional[TesFileType] = TesFileType.FILE + type: Optional[TesFileType] = Field(default=TesFileType.FILE) content: Optional[str] = Field( default=None, description="File content literal.\n\nImplementations should support a minimum " @@ -486,7 +486,7 @@ class TesTask(BaseModel): description="Task identifier assigned by the server.", example="job-0012345", ) - state: Optional[TesState] = TesState.UNKNOWN + state: Optional[TesState] = Field(default=TesState.UNKNOWN) name: Optional[str] = Field(None, description="User-provided task name.") description: Optional[str] = Field( default=None, From 9d643c471cda43399902255d2fc04304cbaf45d6 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 4 Jul 2024 00:33:09 +0530 Subject: [PATCH 10/11] ignore bandit error, because it thinks we have hardcoded temp file dir but its just used as example, so its a false positive. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 1e09c08e..92cd2d18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ requires = ["poetry-core"] skips = [ "B321", # FTP-related functions are being called. "B402", # A FTP-related module is being imported. + "B108" # Insecure usage of temp file/directory, false positive. ] [tool.poetry] From 4bb1dba1e1f486200ada2fa042e0b5a3ace9b5f8 Mon Sep 17 00:00:00 2001 From: Javed Habib Date: Thu, 4 Jul 2024 00:40:59 +0530 Subject: [PATCH 11/11] pydantic is main dep, so that group needs to be installed as well --- .github/workflows/code_quality.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_quality.yaml b/.github/workflows/code_quality.yaml index f3084be4..643f0c2f 100644 --- a/.github/workflows/code_quality.yaml +++ b/.github/workflows/code_quality.yaml @@ -58,8 +58,8 @@ jobs: with: os: ${{ job.os }} python-version: '3.11' - poetry-install-options: "--only=types --no-root" - poetry-export-options: "--only=types" + poetry-install-options: "--with=types --no-root" + poetry-export-options: "--with=types" - name: Check types run: poetry run mypy tesk/