From 72210deabef76c8454950b1de265e665e292b65f Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Fri, 21 Jun 2024 10:02:20 -0700 Subject: [PATCH 1/4] specify OUTPUT_S3_PREFIX and update test_file_mutations.py --- Makefile | 2 +- docker-compose-dev.yml | 1 + docker-compose.yml | 1 + test_app/Makefile | 2 +- test_app/docker-compose.yml | 1 + test_app/tests/test_file_mutations.py | 4 ++-- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 333117b..9df2748 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ build: # Export poetry dependency list as a requirements.txt, which makes Docker builds # faster by not having to reinstall all dependencies every time we build a new wheel. poetry export --without-hashes --format=requirements.txt > requirements.txt - $(docker_compose) build + $(docker_compose) build --no-cache $(MAKE_TEST_APP) build rm requirements.txt diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index cd34057..cc8138b 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -22,6 +22,7 @@ services: - PLATFORMICS_DATABASE_NAME=platformics - DEFAULT_UPLOAD_BUCKET=local-bucket - DEFAULT_UPLOAD_PROTOCOL=s3 + - OUTPUT_S3_PREFIX=nextgen - BOTO_ENDPOINT_URL=http://motoserver.platformics:4000 - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test diff --git a/docker-compose.yml b/docker-compose.yml index b270fb5..8decc74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ services: - PLATFORMICS_DATABASE_NAME=platformics - DEFAULT_UPLOAD_BUCKET=local-bucket - DEFAULT_UPLOAD_PROTOCOL=s3 + - OUTPUT_S3_PREFIX=nextgen - BOTO_ENDPOINT_URL=http://motoserver.platformics:4000 - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test diff --git a/test_app/Makefile b/test_app/Makefile index 656dca9..dca3634 100644 --- a/test_app/Makefile +++ b/test_app/Makefile @@ -60,7 +60,7 @@ clean: ## Remove all codegen'd artifacts. rm -rf cerbos rm -rf support rm -rf database - rm -f .moto_recording + rm -rf .moto_recording rm -rf test_infra $(docker_compose) --profile '*' down diff --git a/test_app/docker-compose.yml b/test_app/docker-compose.yml index 32c81ca..c92d257 100644 --- a/test_app/docker-compose.yml +++ b/test_app/docker-compose.yml @@ -62,6 +62,7 @@ services: - PLATFORMICS_DATABASE_NAME=platformics - DEFAULT_UPLOAD_BUCKET=local-bucket - DEFAULT_UPLOAD_PROTOCOL=s3 + - OUTPUT_S3_PREFIX=nextgen - BOTO_ENDPOINT_URL=http://motoserver.platformics:4000 - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test diff --git a/test_app/tests/test_file_mutations.py b/test_app/tests/test_file_mutations.py index 16d944b..dcb8992 100644 --- a/test_app/tests/test_file_mutations.py +++ b/test_app/tests/test_file_mutations.py @@ -189,7 +189,7 @@ async def test_create_file( # Upload a fastq file to a mock bucket so we can create a file object from it file_namespace = "local-bucket" file_path = "test1.fastq" - file_path_local = "tests/fixtures/test1.fasta" + file_path_local = "tests/fixtures/test1.fastq" file_size = os.stat(file_path_local).st_size with open(file_path_local, "rb") as fp: moto_client.put_object(Bucket=file_namespace, Key=file_path, Body=fp) @@ -211,7 +211,7 @@ async def test_create_file( path size }} - + }} """ output = await gql_client.query(mutation, member_projects=[123], service_identity="workflows") assert output["data"]["createFile"]["size"] == file_size From 8f6c4977c7a07a0582841d9d860a3186cd3ea552 Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Fri, 21 Jun 2024 10:51:39 -0700 Subject: [PATCH 2/4] remove mutable: false --- test_app/schema/schema.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/test_app/schema/schema.yaml b/test_app/schema/schema.yaml index 2667a6f..59fd91e 100644 --- a/test_app/schema/schema.yaml +++ b/test_app/schema/schema.yaml @@ -342,5 +342,4 @@ classes: range: string required: true annotations: - mutable: false plural: ImmutableTypes From 6dc19cb6270da0101044318a02093c316d4e97b4 Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Fri, 21 Jun 2024 11:06:15 -0700 Subject: [PATCH 3/4] lint --- test_app/tests/test_field_visibility.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test_app/tests/test_field_visibility.py b/test_app/tests/test_field_visibility.py index 8a0e5e7..d307c83 100644 --- a/test_app/tests/test_field_visibility.py +++ b/test_app/tests/test_field_visibility.py @@ -130,7 +130,9 @@ async def test_update_fields( ] fields = [field["name"] for field in create_type["inputFields"]] # We have a limited subset of mutable fields on SequencingRead - assert set(fields) == set(["nucleicAcid", "sampleId", "technology", "protocol", "deletedAt", "primerFileId", "collectionId"]) + assert set(fields) == set( + ["nucleicAcid", "sampleId", "technology", "protocol", "deletedAt", "primerFileId", "collectionId"] + ) # Make sure we only allow certain fields to be set at entity creation time. From 509aa517f5fcfaf36dfc16df876dd010021eb4b0 Mon Sep 17 00:00:00 2001 From: Omar Valenzuela Date: Fri, 21 Jun 2024 11:08:25 -0700 Subject: [PATCH 4/4] replace nextgen with platformics --- docker-compose-dev.yml | 2 +- docker-compose.yml | 2 +- platformics/database/models/file.py | 2 +- test_app/docker-compose.yml | 2 +- test_app/tests/test_file_mutations.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index cc8138b..2128258 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -22,7 +22,7 @@ services: - PLATFORMICS_DATABASE_NAME=platformics - DEFAULT_UPLOAD_BUCKET=local-bucket - DEFAULT_UPLOAD_PROTOCOL=s3 - - OUTPUT_S3_PREFIX=nextgen + - OUTPUT_S3_PREFIX=platformics - BOTO_ENDPOINT_URL=http://motoserver.platformics:4000 - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test diff --git a/docker-compose.yml b/docker-compose.yml index 8decc74..636702b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: - PLATFORMICS_DATABASE_NAME=platformics - DEFAULT_UPLOAD_BUCKET=local-bucket - DEFAULT_UPLOAD_PROTOCOL=s3 - - OUTPUT_S3_PREFIX=nextgen + - OUTPUT_S3_PREFIX=platformics - BOTO_ENDPOINT_URL=http://motoserver.platformics:4000 - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test diff --git a/platformics/database/models/file.py b/platformics/database/models/file.py index 8e3ca1a..0ac4d77 100644 --- a/platformics/database/models/file.py +++ b/platformics/database/models/file.py @@ -81,7 +81,7 @@ def before_delete(mapper: Mapper, connection: Connection, target: File) -> None: settings = File.get_settings() s3_client = File.get_s3_client() - # If this file is managed by NextGen, see if it needs to be deleted from S3 + # If this file is managed by platformics, see if it needs to be deleted from S3 if target.path.startswith(f"{settings.OUTPUT_S3_PREFIX}/") and target.protocol == FileAccessProtocol.s3: # Is this the last File object pointing to this path? files_pointing_to_same_path = connection.execute( diff --git a/test_app/docker-compose.yml b/test_app/docker-compose.yml index c92d257..0615a28 100644 --- a/test_app/docker-compose.yml +++ b/test_app/docker-compose.yml @@ -62,7 +62,7 @@ services: - PLATFORMICS_DATABASE_NAME=platformics - DEFAULT_UPLOAD_BUCKET=local-bucket - DEFAULT_UPLOAD_PROTOCOL=s3 - - OUTPUT_S3_PREFIX=nextgen + - OUTPUT_S3_PREFIX=platformics - BOTO_ENDPOINT_URL=http://motoserver.platformics:4000 - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test diff --git a/test_app/tests/test_file_mutations.py b/test_app/tests/test_file_mutations.py index dcb8992..6642075 100644 --- a/test_app/tests/test_file_mutations.py +++ b/test_app/tests/test_file_mutations.py @@ -221,9 +221,9 @@ async def test_create_file( @pytest.mark.parametrize( "file_path,multiple_files_for_one_path,should_delete", [ - ("nextgen/test1.fastq", False, True), + ("platformics/test1.fastq", False, True), ("bla/test1.fastq", False, False), - ("nextgen/test1.fastq", True, False), + ("platformics/test1.fastq", True, False), ("bla/test1.fastq", True, False), ], )