From 0d815d479bd53102b7cd9e1bc66a2048c1aa6df3 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 10 May 2024 11:36:01 -0700
Subject: [PATCH 01/38] missed a clause

---
 permissions_engine/permissions.rego | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/permissions_engine/permissions.rego b/permissions_engine/permissions.rego
index 7065ded..69abfcb 100644
--- a/permissions_engine/permissions.rego
+++ b/permissions_engine/permissions.rego
@@ -12,8 +12,8 @@ package permissions
 #     'program': name of program (optional)
 # }
 #
-import data.idp.valid_token
-import data.idp.user_key
+import data.idp.valid_token as valid_token
+import data.idp.user_key as user_key
 import future.keywords.in
 
 #

From aa828e09875e8b7a69492655e818041ad59ba4c1 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 10 May 2024 17:24:39 -0700
Subject: [PATCH 02/38] Update entrypoint.sh

---
 entrypoint.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/entrypoint.sh b/entrypoint.sh
index 9b4e94c..59740de 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -4,9 +4,6 @@ set -Euo pipefail
 
 OPA_ROOT_TOKEN=$(cat /run/secrets/opa-root-token)
 OPA_SERVICE_TOKEN=$(cat /run/secrets/opa-service-token)
-SITE_ADMIN_USER=$(cat /run/secrets/site_admin_name)
-USER1=$(cat /run/secrets/user1_name)
-USER2=$(cat /run/secrets/user2_name)
 
 if [[ -f "/app/initial_setup" ]]; then
     # set up our default values

From 11c1c1e3a49e5a94ba24542731be22fe2c135f8d Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Wed, 15 May 2024 17:19:00 -0700
Subject: [PATCH 03/38] If vault stores already exist, don't wipe them out

---
 initialize_vault_store.py | 43 +++++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/initialize_vault_store.py b/initialize_vault_store.py
index f0fa196..3632170 100644
--- a/initialize_vault_store.py
+++ b/initialize_vault_store.py
@@ -1,6 +1,6 @@
 import json
 import os
-from authx.auth import get_service_store_secret, set_service_store_secret, add_provider_to_opa, add_program_to_opa
+from authx.auth import get_service_store_secret, set_service_store_secret, add_provider_to_opa, add_program_to_opa, list_programs_in_opa
 import sys
 
 ## Initializes Vault's opa service store with the information for our IDP and the data in site_roles.json, paths.json, programs.json
@@ -11,34 +11,41 @@
     with open('/app/bearer.txt') as f:
         try:
             token = f.read().strip()
-            response, status_code = set_service_store_secret("opa", key="data", value=json.dumps({"keys":[]}))
             response = add_provider_to_opa(token, os.getenv("KEYCLOAK_REALM_URL"))
             results.append(response)
         except Exception as e:
             print(str(e))
             sys.exit(1)
 
-    with open('/app/defaults/paths.json') as f:
-        data = f.read()
-        response, status_code = set_service_store_secret("opa", key="paths", value=data)
-        if status_code != 200:
-            sys.exit(3)
-        results.append(response)
+    response, status_code = get_service_store_secret("opa", key="paths")
+    if status_code != 200:
+        with open('/app/defaults/paths.json') as f:
+            data = f.read()
+            response, status_code = set_service_store_secret("opa", key="paths", value=data)
+            if status_code != 200:
+                sys.exit(3)
+            results.append(response)
 
-    with open('/app/defaults/site_roles.json') as f:
-        data = f.read()
-        response, status_code = set_service_store_secret("opa", key="site_roles", value=data)
-        if status_code != 200:
-            sys.exit(2)
-        results.append(response)
+    response, status_code = get_service_store_secret("opa", key="site_roles")
+    if status_code != 200:
+        with open('/app/defaults/site_roles.json') as f:
+            data = f.read()
+            response, status_code = set_service_store_secret("opa", key="site_roles", value=data)
+            if status_code != 200:
+                sys.exit(2)
+            results.append(response)
 
+    current_programs, status_code = list_programs_in_opa()
+    if status_code != 200:
+        current_programs = []
     with open('/app/defaults/programs.json') as f:
         programs = json.load(f)
         for program in programs:
-            response, status_code = add_program_to_opa(programs[program])
-            if status_code != 200:
-                sys.exit(2)
-            results.append(response)
+            if programs[program] not in current_programs:
+                response, status_code = add_program_to_opa(programs[program])
+                if status_code != 200:
+                    sys.exit(2)
+                results.append(response)
 except Exception as e:
     print(str(e))
     sys.exit(4)

From 8c44cb6222c41f96a4e973908467202bfd6993d4 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 16 May 2024 18:41:07 -0700
Subject: [PATCH 04/38] better error messaging

---
 initialize_vault_store.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/initialize_vault_store.py b/initialize_vault_store.py
index 3632170..5dfef6b 100644
--- a/initialize_vault_store.py
+++ b/initialize_vault_store.py
@@ -14,7 +14,7 @@
             response = add_provider_to_opa(token, os.getenv("KEYCLOAK_REALM_URL"))
             results.append(response)
         except Exception as e:
-            print(str(e))
+            raise Exception(f"failed to save idp keys: {str(e)} {status_code}")
             sys.exit(1)
 
     response, status_code = get_service_store_secret("opa", key="paths")
@@ -23,7 +23,7 @@
             data = f.read()
             response, status_code = set_service_store_secret("opa", key="paths", value=data)
             if status_code != 200:
-                sys.exit(3)
+                raise Exception(f"failed to save paths: {str(e)} {status_code}")
             results.append(response)
 
     response, status_code = get_service_store_secret("opa", key="site_roles")
@@ -32,7 +32,7 @@
             data = f.read()
             response, status_code = set_service_store_secret("opa", key="site_roles", value=data)
             if status_code != 200:
-                sys.exit(2)
+                raise Exception(f"failed to save site roles: {str(e)} {status_code}")
             results.append(response)
 
     current_programs, status_code = list_programs_in_opa()
@@ -44,7 +44,7 @@
             if programs[program] not in current_programs:
                 response, status_code = add_program_to_opa(programs[program])
                 if status_code != 200:
-                    sys.exit(2)
+                    raise Exception(f"failed to save program authz: {str(e)} {status_code}")
                 results.append(response)
 except Exception as e:
     print(str(e))

From cdde1c934badfb693bf8bb7b1c4bc56e49cca7d1 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 21 May 2024 20:24:47 -0700
Subject: [PATCH 05/38] generalize decode_verify_token_output

should be able to use either input.identity or input.token
---
 permissions_engine/idp.rego | 41 +++++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/permissions_engine/idp.rego b/permissions_engine/idp.rego
index c82ecd2..dddc374 100644
--- a/permissions_engine/idp.rego
+++ b/permissions_engine/idp.rego
@@ -8,13 +8,15 @@ package idp
 import data.vault.keys as keys
 import future.keywords.in
 
-decode_verify_token_output[issuer] := output {
-    some i
-    issuer := keys[i].iss
-    cert := keys[i].cert
-    aud := keys[i].aud[_]
+#
+# Function to decode and verify if a token is valid against a key
+#
+decode_verify_token(key, token) := output {
+    issuer := key.iss
+    cert := key.cert
+    aud := key.aud[_]
     output := io.jwt.decode_verify(     # Decode and verify in one-step
-        input.token,
+        token,
         {                         # With the supplied constraints:
             "cert": cert,
             "iss": issuer,
@@ -23,6 +25,24 @@ decode_verify_token_output[issuer] := output {
     )
 }
 
+#
+# If either input.identity or input.token are valid against an issuer, decode and verify
+#
+decode_verify_token_output[issuer] := output {
+    possible_tokens := ["identity", "token"]
+    some i
+    issuer := keys[i].iss
+    output := decode_verify_token(keys[i], input[possible_tokens[_]])
+}
+
+#
+# The issuer of this token
+#
+token_issuer := i {
+    some i in object.keys(decode_verify_token_output)
+    decode_verify_token_output[i][0] == true
+}
+
 #
 # Check if token is valid by checking whether decoded_verify output exists or not
 #
@@ -30,7 +50,10 @@ valid_token = true {
     decode_verify_token_output[_][0]
 }
 
-user_key := decode_verify_token_output[_][2].CANDIG_USER_KEY        # get user key from the token payload
+#
+# The user's key, as determined by this candig instance
+#
+user_key := decode_verify_token_output[token_issuer][2].CANDIG_USER_KEY
 
 #
 # Check trusted_researcher in the token payload
@@ -40,8 +63,8 @@ trusted_researcher = true {
 }
 
 #
-# If the issuer in the token is the same as the first listed in keys, this is issued by the local issuer
+# If the token_issuer is the same as the first listed in keys, this is a local token
 #
 is_local_token = true {
-    keys[i].iss in object.keys(decode_verify_token_output)
+    keys[0].iss == token_issuer
 }

From 09b77d9fde8af86b3aedc7b1f572ee6fa1fccb70 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 21 May 2024 20:25:06 -0700
Subject: [PATCH 06/38] now site admin can be calculated on authz.rego

---
 permissions_engine/authz.rego | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index 0ae0dd7..0743566 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -69,3 +69,8 @@ allow {
     input.path == ["v1", "data", "service", "service-info"]
     input.method == "GET"
 }
+
+# Site admin should be able to see anything
+allow {
+    data.permissions.site_admin == true
+}

From 36a1cdcf1e4a99387ae99f9aaa8e9d80485d4b7c Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 21 May 2024 21:29:42 -0700
Subject: [PATCH 07/38] authorized user should be able to view datasets

---
 permissions_engine/authz.rego | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index 0743566..31e3597 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -74,3 +74,11 @@ allow {
 allow {
     data.permissions.site_admin == true
 }
+
+# As long as the user is authorized, should be able to get their own datasets
+allow {
+    input.path == ["v1", "data", "permissions", "datasets"]
+    input.method == "POST"
+    data.permissions.valid_token == true
+    input.body.input.token == input.identity
+}

From 577a43628a068ea79f7e08d05a26da8825257c0a Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 21 May 2024 21:52:46 -0700
Subject: [PATCH 08/38] also user can view "allowed"

---
 permissions_engine/authz.rego | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index 31e3597..2e8d148 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -82,3 +82,11 @@ allow {
     data.permissions.valid_token == true
     input.body.input.token == input.identity
 }
+
+# As long as the user is authorized, should be able to see if they're allowed to view something
+allow {
+    input.path == ["v1", "data", "permissions", "allowed"]
+    input.method == "POST"
+    data.permissions.valid_token == true
+    input.body.input.token == input.identity
+}

From dff52b1ead066fb158a746b841ae194285a79c0a Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Wed, 22 May 2024 20:08:18 -0700
Subject: [PATCH 09/38] opa can generate its own secret for storing its vault
 token

---
 entrypoint.sh                 | 7 ++-----
 get_vault_store_token.py      | 6 +++---
 permissions_engine/authz.rego | 7 +++++++
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/entrypoint.sh b/entrypoint.sh
index 59740de..5c4fcf7 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -2,8 +2,6 @@
 
 set -Euo pipefail
 
-OPA_ROOT_TOKEN=$(cat /run/secrets/opa-root-token)
-OPA_SERVICE_TOKEN=$(cat /run/secrets/opa-service-token)
 
 if [[ -f "/app/initial_setup" ]]; then
     # set up our default values
@@ -17,9 +15,8 @@ if [[ -f "/app/initial_setup" ]]; then
     sed -i s/USER1/$USER1/ /app/defaults/programs.json
     sed -i s/USER2/$USER2/ /app/defaults/programs.json
 
-    sed -i s/OPA_SERVICE_TOKEN/$OPA_SERVICE_TOKEN/ /app/permissions_engine/authz.rego
-    sed -i s/OPA_ROOT_TOKEN/$OPA_ROOT_TOKEN/ /app/permissions_engine/authz.rego
-
+    token=$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | base64 | tr -d '\n\r+' | sed s/[^A-Za-z0-9]//g)
+    echo { \"opa_secret\": \"$token\" } > /app/permissions_engine/opa_secret.json
     # set up vault URL
     sed -i s@VAULT_URL@$VAULT_URL@ /app/permissions_engine/vault.rego
 
diff --git a/get_vault_store_token.py b/get_vault_store_token.py
index 7cda194..e88e1c5 100644
--- a/get_vault_store_token.py
+++ b/get_vault_store_token.py
@@ -7,11 +7,11 @@
 
 # get the token for the opa store
 try:
-    with open("/run/secrets/opa-root-token") as f:
-        OPA_ROOT_TOKEN = f.read().strip()
+    with open("/app/permissions_engine/opa_secret.json") as f:
+        opa_json = json.load(f)
         opa_token = get_vault_token_for_service("opa")
         headers = {
-            "X-Opa": OPA_ROOT_TOKEN,
+            "X-Opa": opa_json["opa_secret"],
             "Content-Type": "application/json; charset=utf-8"
         }
         payload = f"{{\"token\": \"{opa_token}\"}}"
diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index 2e8d148..bb82f57 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -64,6 +64,13 @@ allow {
     input.method == "POST"
 }
 
+# Opa should be able to store its vault token
+allow {
+    input.path == ["v1", "data", "store_token"]
+    input.method == "PUT"
+    input.headers["X-Opa"][_] == data.opa_secret
+}
+
 # Service-info path for healthcheck
 allow {
     input.path == ["v1", "data", "service", "service-info"]

From 30b2af8412db02fb75261144950c669f8bd6874c Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 23 May 2024 12:07:57 -0700
Subject: [PATCH 10/38] remove token-based stuff

---
 permissions_engine/authz.rego | 51 -----------------------------------
 1 file changed, 51 deletions(-)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index bb82f57..ebd3300 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -3,59 +3,8 @@ package system.authz
 # this defines authentication to have access to opa at all
 # from: https://www.openpolicyagent.org/docs/v0.22.0/security/#token-based-authentication-example
 
-rights = {
-    "admin": {
-        "path": "*"
-    },
-    "datasets": {
-        "path": ["v1", "data", "permissions", "datasets"]
-    },
-    "allowed": {
-        "path": ["v1", "data", "permissions", "allowed"]
-    },
-    "site_admin": {
-        "path": ["v1", "data", "permissions", "site_admin"]
-    },
-    "user_id": {
-        "path": ["v1", "data", "idp", "user_key"]
-    },
-    "tokenControlledAccessREMS": {
-        "path": ["v1", "data", "ga4ghPassport", "tokenControlledAccessREMS"]
-    }
-}
-
-root_token := "OPA_ROOT_TOKEN"
-service_token := "OPA_SERVICE_TOKEN"
-
-tokens = {
-    root_token : {
-        "roles": ["admin"]
-    },
-    service_token : {
-        "roles": ["datasets", "allowed", "site_admin", "user_id", "tokenControlledAccessREMS"]
-    }
-}
-
 default allow = false               # Reject requests by default.
 
-allow {                             # Allow request if...
-    some right
-    identity_rights[right]          # Rights for identity exist, and...
-    right.path == "*"               # Right.path is '*'.
-}
-
-allow {                             # Allow request if...
-    some right
-    identity_rights[right]          # Rights for identity exist, and...
-    right.path == input.path        # Right.path matches input.path.
-}
-
-x_opa := input.headers["X-Opa"][_]
-
-identity_rights[right] {             # Right is in the identity_rights set if...
-    token := tokens[x_opa]  # Token exists for identity, and...
-    role := token.roles[_]           # Token has a role, and...
-    right := rights[role]            # Role has rights defined.
 }
 
 # Any service should be able to verify that a service is who it says it is:

From 9256ea627089b872cf37d3d06ed5f5a671722b06 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 23 May 2024 12:09:23 -0700
Subject: [PATCH 11/38] replace old rights with access for a user's own data
 only

---
 permissions_engine/authz.rego | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index ebd3300..cea4eb1 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -31,17 +31,18 @@ allow {
     data.permissions.site_admin == true
 }
 
-# As long as the user is authorized, should be able to get their own datasets
-allow {
-    input.path == ["v1", "data", "permissions", "datasets"]
-    input.method == "POST"
-    data.permissions.valid_token == true
-    input.body.input.token == input.identity
+# The authx library uses these paths:
+authx_paths = {
+    "datasets": ["v1", "data", "permissions", "datasets"],
+    "allowed": ["v1", "data", "permissions", "allowed"],
+    "site_admin": ["v1", "data", "permissions", "site_admin"],
+    "user_id": ["v1", "data", "idp", "user_key"]
 }
 
-# As long as the user is authorized, should be able to see if they're allowed to view something
+# An authorized user has a valid token (and passes in that same token for both bearer and body)
+# Authz users can access the authx paths
 allow {
-    input.path == ["v1", "data", "permissions", "allowed"]
+    input.path == authx_paths[_]
     input.method == "POST"
     data.permissions.valid_token == true
     input.body.input.token == input.identity

From 60bcc4ce2fa6c12e496fb8ad0afb8adee984a161 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 23 May 2024 12:10:10 -0700
Subject: [PATCH 12/38] reordering

---
 permissions_engine/authz.rego | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index cea4eb1..91fa6c9 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -3,8 +3,12 @@ package system.authz
 # this defines authentication to have access to opa at all
 # from: https://www.openpolicyagent.org/docs/v0.22.0/security/#token-based-authentication-example
 
-default allow = false               # Reject requests by default.
+# Reject requests by default
+default allow = false
 
+# Site admin should be able to see anything
+allow {
+    data.permissions.site_admin == true
 }
 
 # Any service should be able to verify that a service is who it says it is:
@@ -26,11 +30,6 @@ allow {
     input.method == "GET"
 }
 
-# Site admin should be able to see anything
-allow {
-    data.permissions.site_admin == true
-}
-
 # The authx library uses these paths:
 authx_paths = {
     "datasets": ["v1", "data", "permissions", "datasets"],

From 84043eb0dc2816f3f22555392d4914c41b26d3ea Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Mon, 27 May 2024 11:23:24 -0700
Subject: [PATCH 13/38] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e17ac12..43892e0 100644
--- a/README.md
+++ b/README.md
@@ -9,13 +9,13 @@ Interactions with Vault are handled by [vault.rego](permissions_engine/vault.reg
 
 Authorization to endpoints in the OPA service itself is defined in [authz.rego](permissions_engine/authz.rego).
 
-* Token-based auth: There are two api tokens defined: the root token allows any path to be accessed, while the service token only allows the `permissions/datasets` and `permissions/allowed` endpoints to be viewed.
-
 * Role-based auth: Roles for the site are defined in the format given in [site_roles.json](defaults/site_roles.json). if the User is defined as a site admin, they are allowed to view any endpoint. Other site-based roles can be similarly defined.
 
 * Endpoint-based auth: Any service can use the `/service/verified` endpoint. Other specific endpoints can be similarly allowed.
 
-* Program-based and user-based authorizations are defined at the `permissions` path: For a given User and the method of accessing a service (method, path), the `/permissions/datasets` endpoint returns the list of programs that user is allowed to access for that method/path, while the `/permissions/allowed` endpoint returns True if either the user is a site admin or the user is allowed to access that method/path. The following two types of authorizations are available:
+* An authenticated and authorized user is allowed to find out their own user ID, the key of which is defined system-wide in the .env file as CANDIG_USER_KEY. By default, this is the user's email address. This is the user ID by which user-based and program-based authorizations are keyed.
+
+* Program-based and user-based authorizations are defined at the `permissions` path: A User can access these Opa endpoints to introspect their own authorizations. For a given method of accessing a service (method, path), the `/permissions/datasets` endpoint returns the list of programs that the User is allowed to access for that method/path, while the `/permissions/allowed` endpoint returns True if either the User is a site admin or the User is allowed to access that method/path. The following two types of authorizations are available:
 
   * Authorizations for roles in particular programs: users defined as team_members for a program are allowed to access the read paths specified in [paths.json](defaults/paths.json), while users defined as program_curators are allowed to access the curate and delete paths. Note: read and curate paths are separately allowed: if a user should be allowed to both read and curate, they should be in both the team_members and program_curators groups. Program authorizations can be created, edited, and deleted through the ingest microservice. Default test examples can be found in [programs.json](defaults/programs.json).
 

From 589341f780aa8b0df7291993533fa597f35b4d4f Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Mon, 27 May 2024 11:31:27 -0700
Subject: [PATCH 14/38] Update requirements.txt

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f33ddd7..c5506a6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 requests
 jq
 pytest==7.2.0
-candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.3.0
+candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.2
 

From 65327a32fa5d4e37c3404b6cba3fd9dc31be9d6a Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 31 May 2024 17:36:39 -0700
Subject: [PATCH 15/38] pick up renew-idp changes in authx

---
 requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index c5506a6..c78e1c1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,4 @@
 requests
 jq
 pytest==7.2.0
-candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.2
-
+candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@daisieh/renew-idp

From 02d41800923de3cad3456f2d1f7ffbd4a7ea1e98 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 31 May 2024 17:36:59 -0700
Subject: [PATCH 16/38] separate idp and other vault stores

---
 initialize_idp.py         | 23 +++++++++++++++++++++++
 initialize_vault_store.py | 19 +++++--------------
 2 files changed, 28 insertions(+), 14 deletions(-)
 create mode 100644 initialize_idp.py

diff --git a/initialize_idp.py b/initialize_idp.py
new file mode 100644
index 0000000..baa3b4d
--- /dev/null
+++ b/initialize_idp.py
@@ -0,0 +1,23 @@
+import json
+import os
+from authx.auth import add_provider_to_opa, get_user_id
+import sys
+
+## Updates Vault's opa service store with the information for our IDP
+
+token = None
+try:
+    if os.path.isfile('/app/bearer.txt'):
+        with open('/app/bearer.txt') as f:
+            token = f.read().strip()
+    if token is not None:
+        response = add_provider_to_opa(token, os.getenv("KEYCLOAK_REALM_URL"))
+        os.remove('/app/bearer.txt')
+        if get_user_id(None, token=token) is None:
+            print("IDP is incorrect: verify that Keycloak is set up and clean/build/compose opa again")
+            sys.exit(2)
+except Exception as e:
+    raise Exception(f"failed to save idp keys: {str(e)} {status_code}")
+    sys.exit(1)
+
+sys.exit(0)
diff --git a/initialize_vault_store.py b/initialize_vault_store.py
index 5dfef6b..ea629f4 100644
--- a/initialize_vault_store.py
+++ b/initialize_vault_store.py
@@ -1,29 +1,20 @@
 import json
 import os
-from authx.auth import get_service_store_secret, set_service_store_secret, add_provider_to_opa, add_program_to_opa, list_programs_in_opa
+from authx.auth import get_service_store_secret, set_service_store_secret, add_program_to_opa, list_programs_in_opa
 import sys
 
-## Initializes Vault's opa service store with the information for our IDP and the data in site_roles.json, paths.json, programs.json
+## Initializes Vault's opa service store with the data in site_roles.json, paths.json, programs.json
 
 results = []
 
 try:
-    with open('/app/bearer.txt') as f:
-        try:
-            token = f.read().strip()
-            response = add_provider_to_opa(token, os.getenv("KEYCLOAK_REALM_URL"))
-            results.append(response)
-        except Exception as e:
-            raise Exception(f"failed to save idp keys: {str(e)} {status_code}")
-            sys.exit(1)
-
     response, status_code = get_service_store_secret("opa", key="paths")
     if status_code != 200:
         with open('/app/defaults/paths.json') as f:
             data = f.read()
             response, status_code = set_service_store_secret("opa", key="paths", value=data)
             if status_code != 200:
-                raise Exception(f"failed to save paths: {str(e)} {status_code}")
+                raise Exception(f"failed to save paths: {response} {status_code}")
             results.append(response)
 
     response, status_code = get_service_store_secret("opa", key="site_roles")
@@ -32,7 +23,7 @@
             data = f.read()
             response, status_code = set_service_store_secret("opa", key="site_roles", value=data)
             if status_code != 200:
-                raise Exception(f"failed to save site roles: {str(e)} {status_code}")
+                raise Exception(f"failed to save site roles: {response} {status_code}")
             results.append(response)
 
     current_programs, status_code = list_programs_in_opa()
@@ -44,7 +35,7 @@
             if programs[program] not in current_programs:
                 response, status_code = add_program_to_opa(programs[program])
                 if status_code != 200:
-                    raise Exception(f"failed to save program authz: {str(e)} {status_code}")
+                    raise Exception(f"failed to save program authz: {response} {status_code}")
                 results.append(response)
 except Exception as e:
     print(str(e))

From 888f2d93a7f08f58f95d865bd8ea64b7dc0fe1d9 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 31 May 2024 17:37:14 -0700
Subject: [PATCH 17/38] always initialize idp

---
 entrypoint.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/entrypoint.sh b/entrypoint.sh
index 5c4fcf7..e3f20d3 100644
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -24,13 +24,15 @@ if [[ -f "/app/initial_setup" ]]; then
     python3 /app/initialize_vault_store.py
     if [[ $? -eq 0 ]]; then
         rm /app/initial_setup
-        rm /app/bearer.txt
         echo "setup complete"
     else
         echo "!!!!!! INITIALIZATION FAILED, TRY AGAIN !!!!!!"
     fi
 fi
 
+# make sure that our idp is still set correctly (maybe keycloak was reinitialized)
+python3 get_vault_store_token.py
+python3 /app/initialize_idp.py
 
 while [ 0 -eq 0 ]
 do

From 9a13df5b869010740981bdbbf863e47f7a151cba Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 31 May 2024 18:02:17 -0700
Subject: [PATCH 18/38] print message

---
 initialize_idp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/initialize_idp.py b/initialize_idp.py
index baa3b4d..e4a272d 100644
--- a/initialize_idp.py
+++ b/initialize_idp.py
@@ -11,6 +11,7 @@
         with open('/app/bearer.txt') as f:
             token = f.read().strip()
     if token is not None:
+        print("Updating our IDP with a new bearer token")
         response = add_provider_to_opa(token, os.getenv("KEYCLOAK_REALM_URL"))
         os.remove('/app/bearer.txt')
         if get_user_id(None, token=token) is None:

From 78434d256245e00017dd849267b48fa4df713f20 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Mon, 3 Jun 2024 11:13:35 -0700
Subject: [PATCH 19/38] Update requirements.txt

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index c78e1c1..bec29dc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 requests
 jq
 pytest==7.2.0
-candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@daisieh/renew-idp
+candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.3

From 70a91d6d0ae47f05011ddf613f5b7094f4f19538 Mon Sep 17 00:00:00 2001
From: Marion <mshadbolt@users.noreply.github.com>
Date: Fri, 7 Jun 2024 17:23:50 -0700
Subject: [PATCH 20/38] allow 'single quote' pr titles

---
 .github/workflows/dispatch-actions.yml | 30 ++++++++++++++------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/dispatch-actions.yml b/.github/workflows/dispatch-actions.yml
index 60a983d..5e31d45 100644
--- a/.github/workflows/dispatch-actions.yml
+++ b/.github/workflows/dispatch-actions.yml
@@ -1,7 +1,8 @@
 name: Submodule PR
 on:
-    push:
+    pull_request:
         branches: [develop]
+        types: [closed]
 jobs:
   CanDIG-dispatch:
     runs-on: ubuntu-latest
@@ -14,17 +15,18 @@ jobs:
             - name: Check out repository code
               uses: actions/checkout@v4
             - name: get PR data
-              uses: actions/github-script@v7
-              id: get_pr_data
-              with:
-                script: |
-                  return (
-                    await github.rest.repos.listPullRequestsAssociatedWithCommit({
-                      commit_sha: context.sha,
-                      owner: context.repo.owner,
-                      repo: context.repo.repo,
-                    })
-                  ).data[0];
+              shell: python
+              run: |
+                  import json
+                  import os
+                  with open('${{ github.event_path }}') as fh:
+                      event = json.load(fh)
+                      escaped = event['pull_request']['title'].replace("'", '"')
+                      pr_number = event["number"]
+                      print(escaped)    
+                  with open(os.environ['GITHUB_ENV'], 'a') as fh:
+                      print(f'PR_TITLE={escaped}', file=fh)
+                      print(f'PR_NUMBER={pr_number}', file=fh)
             - name: Create PR in CanDIGv2
               id: make_pr
               uses: CanDIG/github-action-pr-expanded@v4
@@ -33,7 +35,7 @@ jobs:
                   parent_repository: ${{ env.PARENT_REPOSITORY }}
                   checkout_branch: ${{ env.CHECKOUT_BRANCH}}
                   pr_against_branch: ${{ env.PR_AGAINST_BRANCH }}
-                  pr_title: '${{ github.repository }} merging: ${{ fromJson(steps.get_pr_data.outputs.result).title }}'
-                  pr_description: "PR triggered by update to develop branch on ${{ github.repository }}. Commit hash: `${{ github.sha }}`. PR link: [#${{ fromJson(steps.get_pr_data.outputs.result).number }}](https://github.com/${{ github.repository }}/pull/${{ fromJson(steps.get_pr_data.outputs.result).number }})"
+                  pr_title: "${{ github.repository }} merging: ${{ env.PR_TITLE }}"
+                  pr_description: "PR triggered by update to develop branch on ${{ github.repository }}. Commit hash: `${{ github.sha }}`. PR link: [#${{ env.PR_NUMBER }}](https://github.com/${{ github.repository }}/pull/${{ env.PR_NUMBER }})"
                   owner: ${{ env.OWNER }}
                   submodule_path: lib/opa/opa

From 620ff2d3020142da5208879d43e56931f74c40e9 Mon Sep 17 00:00:00 2001
From: Son Chau <soncqb@gmail.com>
Date: Fri, 9 Aug 2024 12:38:05 -0700
Subject: [PATCH 21/38] change katsu path to v3

---
 defaults/paths.json | 44 ++++++++++++++++++--------------------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/defaults/paths.json b/defaults/paths.json
index 63c60ca..171ac47 100644
--- a/defaults/paths.json
+++ b/defaults/paths.json
@@ -2,16 +2,16 @@
     "paths": {
         "read": {
             "get": [
-                "/v2/discovery/?.*",
-                "/v2/authorized/?.*",
-                "/htsget/v1/variants/?.*",
-                "/htsget/v1/variants/search",
-                "/htsget/v1/reads/?.*",
-                "/htsget/v1/samples/?.*",
-                "/ga4gh/drs/v1/objects/?.*",
-                "/ga4gh/drs/v1/cohorts/?.*",
-                "/ga4gh/drs/v1/cohorts/?.*/status",
-                "/beacon/v2/g_variants/?.*"
+              "/v3/discovery/?.*",
+              "/v3/authorized/?.*",
+              "/htsget/v1/variants/?.*",
+              "/htsget/v1/variants/search",
+              "/htsget/v1/reads/?.*",
+              "/htsget/v1/samples/?.*",
+              "/ga4gh/drs/v1/objects/?.*",
+              "/ga4gh/drs/v1/cohorts/?.*",
+              "/ga4gh/drs/v1/cohorts/?.*/status",
+              "/beacon/v2/g_variants/?.*"
             ],
             "post": [
                 "/htsget/v1/variants/search",
@@ -20,22 +20,14 @@
             ]
         },
         "curate": {
-            "get": [
-                "/htsget/v1/variants/?.*/index",
-                "/htsget/v1/variants/?.*/verify",
-                "/htsget/v1/reads/?.*/index",
-                "/htsget/v1/reads/?.*/verify"
-            ],
-            "post": [
-                "/ingest/?.*",
-                "/ga4gh/drs/v1/?.*",
-                "/v2/ingest/?.*"
-            ],
-            "delete": [
-                "/ingest/?.*",
-                "/ga4gh/drs/v1/?.*",
-                "/v2/ingest/?.*"
-            ]
+          "get": [
+            "/htsget/v1/variants/?.*/index",
+            "/htsget/v1/variants/?.*/verify",
+            "/htsget/v1/reads/?.*/index",
+            "/htsget/v1/reads/?.*/verify"
+          ],
+          "post": ["/ingest/?.*", "/ga4gh/drs/v1/?.*", "/v3/ingest/?.*"],
+          "delete": ["/ingest/?.*", "/ga4gh/drs/v1/?.*", "/v3/ingest/?.*"]
         }
     }
 }
\ No newline at end of file

From 3e52283230986a75ae90de3b7344358af4c05eac Mon Sep 17 00:00:00 2001
From: Son Chau <soncqb@gmail.com>
Date: Fri, 9 Aug 2024 12:38:19 -0700
Subject: [PATCH 22/38] change katsu test path to v3

---
 tests/test_opa_permissions.py | 46 ++++++++++-------------------------
 1 file changed, 13 insertions(+), 33 deletions(-)

diff --git a/tests/test_opa_permissions.py b/tests/test_opa_permissions.py
index fb83b8b..6b5f2fc 100644
--- a/tests/test_opa_permissions.py
+++ b/tests/test_opa_permissions.py
@@ -243,47 +243,27 @@ def test_site_admin(user, expected_result, site_roles, users, programs):
 
 def get_user_datasets():
     return [
-        ( # site admin should be able to read all datasets
+        (  # site admin should be able to read all datasets
             "site_admin",
-            {
-                "body": {
-                  "path": "/ga4gh/drs/v1/cohorts/",
-                  "method": "GET"
-                }
-            },
-            ["SYNTHETIC-1", "SYNTHETIC-2", "SYNTHETIC-3", "SYNTHETIC-4"]
+            {"body": {"path": "/ga4gh/drs/v1/cohorts/", "method": "GET"}},
+            ["SYNTHETIC-1", "SYNTHETIC-2", "SYNTHETIC-3", "SYNTHETIC-4"],
         ),
-        ( # user1 can view the datasets it's a member of
+        (  # user1 can view the datasets it's a member of
             "user1",
-            {
-                "body": {
-                  "path": "/v2/discovery/programs/",
-                  "method": "GET"
-                }
-            },
-            ["SYNTHETIC-1", "SYNTHETIC-3", "SYNTHETIC-4"]
+            {"body": {"path": "/v3/discovery/programs/", "method": "GET"}},
+            ["SYNTHETIC-1", "SYNTHETIC-3", "SYNTHETIC-4"],
         ),
-        ( # user3 can view the datasets it's a member of + DAC programs,
-          # but SYNTHETIC-1's authorized dates are in the past
+        (  # user3 can view the datasets it's a member of + DAC programs,
+            # but SYNTHETIC-1's authorized dates are in the past
             "user3",
-            {
-                "body": {
-                  "path": "/v2/discovery/programs/",
-                  "method": "GET"
-                }
-            },
-            ["SYNTHETIC-3", "SYNTHETIC-4"]
+            {"body": {"path": "/v3/discovery/programs/", "method": "GET"}},
+            ["SYNTHETIC-3", "SYNTHETIC-4"],
         ),
         (
             "dac_user",
-            {
-                "body": {
-                  "path": "/ga4gh/drs/v1/cohorts",
-                  "method": "GET"
-                }
-            },
-            ["SYNTHETIC-3"]
-        )
+            {"body": {"path": "/ga4gh/drs/v1/cohorts", "method": "GET"}},
+            ["SYNTHETIC-3"],
+        ),
     ]
 
 

From e4ec7af7d1219fc2250d5d5153006f6ff9a064b0 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 22 Aug 2024 16:09:47 -0700
Subject: [PATCH 23/38] logging v1.0.0

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index bec29dc..2e25896 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ requests
 jq
 pytest==7.2.0
 candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.3
+candigv2-logging@git+https://github.com/CanDIG/candigv2-logging.git@v1.0.0

From e5c9ff2b403b1aec85e1020793fe45334444d322 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Wed, 28 Aug 2024 10:55:18 -0700
Subject: [PATCH 24/38] Update requirements.txt

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 2e25896..7a49aae 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 requests
 jq
 pytest==7.2.0
-candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.3
+candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.4
 candigv2-logging@git+https://github.com/CanDIG/candigv2-logging.git@v1.0.0

From 306d09bb0a6328ea825b0348b4cd262a31345108 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 17 Sep 2024 17:11:52 -0700
Subject: [PATCH 25/38] add site curator role

---
 permissions_engine/permissions.rego | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/permissions_engine/permissions.rego b/permissions_engine/permissions.rego
index 69abfcb..56a096b 100644
--- a/permissions_engine/permissions.rego
+++ b/permissions_engine/permissions.rego
@@ -101,7 +101,32 @@ else := readable_programs
     regex.match(paths.read.post[_], input.body.path) == true
 }
 
-# if user is a program_curator, they can access programs that allow curate access for this method, path
+# if user is a site curator, they can access all programs that allow curate access for this method, path
+else := all_programs
+{
+    valid_token
+    user_key in site_roles.curator
+    input.body.method = "GET"
+    regex.match(paths.curate.get[_], input.body.path) == true
+}
+
+else := all_programs
+{
+    valid_token
+    user_key in site_roles.curator
+    input.body.method = "POST"
+    regex.match(paths.curate.post[_], input.body.path) == true
+}
+
+else := all_programs
+{
+    valid_token
+    user_key in site_roles.curator
+    input.body.method = "DELETE"
+    regex.match(paths.curate.delete[_], input.body.path) == true
+}
+
+# if user is a program_curator, they can access programs that allow curate access for them for this method, path
 else := curateable_programs
 {
     valid_token

From e73a8ed866a130c8252b7d25eb80e9b64b8bf406 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 17 Sep 2024 17:12:04 -0700
Subject: [PATCH 26/38] add test for site curator

---
 defaults/site_roles.json      |  1 +
 tests/test_opa_permissions.py | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/defaults/site_roles.json b/defaults/site_roles.json
index 6c67c18..0ebaf29 100644
--- a/defaults/site_roles.json
+++ b/defaults/site_roles.json
@@ -4,6 +4,7 @@
             "SITE_ADMIN_USER"
         ],
         "curator": [
+            "USER2"
         ],
         "local_team": [
             "USER1"
diff --git a/tests/test_opa_permissions.py b/tests/test_opa_permissions.py
index 6b5f2fc..a340826 100644
--- a/tests/test_opa_permissions.py
+++ b/tests/test_opa_permissions.py
@@ -23,7 +23,9 @@ def site_roles():
       "admin": [
         "site_admin@test.ca"
       ],
-      "curator": [],
+      "curator": [
+        "user2@test.ca"
+      ],
       "local_team": [
         "user1@test.ca"
       ],
@@ -284,6 +286,17 @@ def get_curation_allowed():
             },
             True
         ),
+        ( # user2 can curate the datasets it's not a curator of because they're a site curator
+            "user2",
+            {
+                "body": {
+                  "path": "/ga4gh/drs/v1/cohorts/",
+                  "method": "POST",
+                  "program": "SYNTHETIC-1"
+                }
+            },
+            True
+        ),
         ( # user1 can curate the datasets it's a curator of
             "user1",
             {

From 3f4ac54eede1ffcab6b0567d3ce096f5609d9216 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 17 Sep 2024 17:13:26 -0700
Subject: [PATCH 27/38] bump authx version

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 7a49aae..0594cb5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 requests
 jq
 pytest==7.2.0
-candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.4
+candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.5
 candigv2-logging@git+https://github.com/CanDIG/candigv2-logging.git@v1.0.0

From 50b95a539e0ec1f6a6605a5adba47bff1f7bf228 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Tue, 17 Sep 2024 17:21:45 -0700
Subject: [PATCH 28/38] Update README.md

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 43892e0..cb12796 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,10 @@ Interactions with Vault are handled by [vault.rego](permissions_engine/vault.reg
 
 Authorization to endpoints in the OPA service itself is defined in [authz.rego](permissions_engine/authz.rego).
 
-* Role-based auth: Roles for the site are defined in the format given in [site_roles.json](defaults/site_roles.json). if the User is defined as a site admin, they are allowed to view any endpoint. Other site-based roles can be similarly defined.
+* Role-based auth: Roles for the site are defined in the format given in [site_roles.json](defaults/site_roles.json).
+  * If the User is defined as a site admin, they are allowed to access any endpoint.
+  * If the User is defined as a site curator, they are allowed to use any of the curate method/path combinations defined in [paths.json](defaults/paths.json) for all programs known to the system.
+  * Other site-based roles can be similarly defined.
 
 * Endpoint-based auth: Any service can use the `/service/verified` endpoint. Other specific endpoints can be similarly allowed.
 

From 1b8961f38e6a3d8c90f346bc66e894f784d127d7 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 20 Sep 2024 18:18:54 -0700
Subject: [PATCH 29/38] move contents of permissions.rego to calculate.rego

---
 permissions_engine/calculate.rego | 144 ++++++++++++++++++++++++++++++
 1 file changed, 144 insertions(+)
 create mode 100644 permissions_engine/calculate.rego

diff --git a/permissions_engine/calculate.rego b/permissions_engine/calculate.rego
new file mode 100644
index 0000000..4077de1
--- /dev/null
+++ b/permissions_engine/calculate.rego
@@ -0,0 +1,144 @@
+package calculate
+#
+# This is the set of policy definitions for the permissions engine.
+#
+
+#
+# Provided:
+# input = {
+#     'token': user token
+#     'method': method requested at data service
+#     'path': path to request at data service
+#     'program': name of program (optional)
+# }
+#
+import data.idp.user_key as user_key
+import future.keywords.in
+
+#
+# This user is a site admin if they have the site_admin role
+#
+import data.vault.site_roles as site_roles
+site_admin = true {
+    user_key in site_roles.admin
+}
+
+#
+# what programs are available to this user?
+#
+
+import data.vault.all_programs as all_programs
+import data.vault.program_auths as program_auths
+import data.vault.user_programs as user_programs
+
+# compile list of programs specifically authorized for the user by DACs and within the authorized time period
+user_readable_programs[p["program_id"]] := output {
+    some p in user_programs
+    time.parse_ns("2006-01-02", p["start_date"]) <= time.now_ns()
+    time.parse_ns("2006-01-02", p["end_date"]) >= time.now_ns()
+    output := p
+}
+
+# compile list of programs that list the user as a team member
+team_readable_programs[p] := output {
+    some p in all_programs
+    user_key in program_auths[p].team_members
+    output := program_auths[p].team_members
+}
+
+# user can read programs that are either team-readable or user-readable
+readable_programs := object.keys(object.union(team_readable_programs, user_readable_programs))
+
+# user can curate programs that list the user as a program curator
+curateable_programs[p] {
+    some p in all_programs
+    user_key in program_auths[p].program_curators
+}
+
+import data.vault.paths as paths
+
+# debugging
+readable_get[p] := output {
+    some p in paths.read.get
+    output := regex.match(p, input.body.path)
+}
+readable_post[p] := output {
+    some p in paths.read.post
+    output := regex.match(p, input.body.path)
+}
+curateable_get[p] := output {
+    some p in paths.curate.get
+    output := regex.match(p, input.body.path)
+}
+curateable_post[p] := output {
+    some p in paths.curate.post
+    output := regex.match(p, input.body.path)
+}
+curateable_delete[p] := output {
+    some p in paths.curate.delete
+    output := regex.match(p, input.body.path)
+}
+
+# which datasets can this user see for this method, path
+default datasets = []
+
+# site admins can see all programs
+datasets := all_programs
+{
+    site_admin
+}
+
+# if user is a team_member, they can access programs that allow read access for this method, path
+else := readable_programs
+{
+    input.body.method = "GET"
+    regex.match(paths.read.get[_], input.body.path) == true
+}
+
+else := readable_programs
+{
+    input.body.method = "POST"
+    regex.match(paths.read.post[_], input.body.path) == true
+}
+
+# if user is a site curator, they can access all programs that allow curate access for this method, path
+else := all_programs
+{
+    user_key in site_roles.curator
+    input.body.method = "GET"
+    regex.match(paths.curate.get[_], input.body.path) == true
+}
+
+else := all_programs
+{
+    user_key in site_roles.curator
+    input.body.method = "POST"
+    regex.match(paths.curate.post[_], input.body.path) == true
+}
+
+else := all_programs
+{
+    user_key in site_roles.curator
+    input.body.method = "DELETE"
+    regex.match(paths.curate.delete[_], input.body.path) == true
+}
+
+# if user is a program_curator, they can access programs that allow curate access for them for this method, path
+else := curateable_programs
+{
+    input.body.method = "GET"
+    regex.match(paths.curate.get[_], input.body.path) == true
+}
+
+else := curateable_programs
+{
+    input.body.method = "POST"
+    regex.match(paths.curate.post[_], input.body.path) == true
+}
+
+else := curateable_programs
+{
+    input.body.method = "DELETE"
+    regex.match(paths.curate.delete[_], input.body.path) == true
+}
+

From a79f5abb0cd9059be4c682471851a26af83d55fa Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 20 Sep 2024 18:19:18 -0700
Subject: [PATCH 30/38] user_key does not need verification

---
 permissions_engine/idp.rego | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/permissions_engine/idp.rego b/permissions_engine/idp.rego
index dddc374..020fe01 100644
--- a/permissions_engine/idp.rego
+++ b/permissions_engine/idp.rego
@@ -25,6 +25,23 @@ decode_verify_token(key, token) := output {
     )
 }
 
+decode_token(token) := output {
+    output := io.jwt.decode(token)
+}
+
+decoded_output := output {
+    possible_tokens := ["identity", "token"]
+    output := decode_token(input[possible_tokens[_]])
+}
+
+user_info := decoded_output[1]
+
+#
+# The user's key, as determined by this candig instance
+#
+user_key := user_info.CANDIG_USER_KEY
+
+
 #
 # If either input.identity or input.token are valid against an issuer, decode and verify
 #
@@ -50,11 +67,6 @@ valid_token = true {
     decode_verify_token_output[_][0]
 }
 
-#
-# The user's key, as determined by this candig instance
-#
-user_key := decode_verify_token_output[token_issuer][2].CANDIG_USER_KEY
-
 #
 # Check trusted_researcher in the token payload
 #

From c647d87ec0fae43b3b9c3be485fbf20bf45dfd07 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 20 Sep 2024 18:21:03 -0700
Subject: [PATCH 31/38] permissions.rego only shows relevant results

---
 permissions_engine/permissions.rego | 174 +++++++---------------------
 1 file changed, 45 insertions(+), 129 deletions(-)

diff --git a/permissions_engine/permissions.rego b/permissions_engine/permissions.rego
index 56a096b..c3aff99 100644
--- a/permissions_engine/permissions.rego
+++ b/permissions_engine/permissions.rego
@@ -1,160 +1,76 @@
 package permissions
-#
-# This is the set of policy definitions for the permissions engine.
-#
-
-#
-# Provided:
-# input = {
-#     'token': user token
-#     'method': method requested at data service
-#     'path': path to request at data service
-#     'program': name of program (optional)
-# }
-#
-import data.idp.valid_token as valid_token
-import data.idp.user_key as user_key
 import future.keywords.in
 
 #
-# This user is a site admin if they have the site_admin role
+# Values that are used by authx
 #
-import data.vault.site_roles as site_roles
-site_admin = true {
-    user_key in site_roles.admin
+valid_token := true {
+    data.idp.valid_token
 }
+else := false
 
-#
-# what programs are available to this user?
-#
-
-import data.vault.all_programs as all_programs
-import data.vault.program_auths as program_auths
-import data.vault.user_programs as user_programs
-
-# compile list of programs specifically authorized for the user by DACs and within the authorized time period
-user_readable_programs[p["program_id"]] := output {
-    some p in user_programs
-    time.parse_ns("2006-01-02", p["start_date"]) <= time.now_ns()
-    time.parse_ns("2006-01-02", p["end_date"]) >= time.now_ns()
-    output := p
-}
-
-# compile list of programs that list the user as a team member
-team_readable_programs[p] := output {
-    some p in all_programs
-    user_key in program_auths[p].team_members
-    output := program_auths[p].team_members
+site_admin := data.calculate.site_admin {
+    valid_token
 }
 
-# user can read programs that are either team-readable or user-readable
-readable_programs := object.keys(object.union(team_readable_programs, user_readable_programs))
-
-# user can curate programs that list the user as a program curator
-curateable_programs[p] {
-    some p in all_programs
-    user_key in program_auths[p].program_curators
+datasets := data.calculate.datasets {
+    valid_token
 }
+else := []
 
-import data.vault.paths as paths
-
-# debugging
-valid_token := valid_token
-readable_get[p] := output {
-    some p in paths.read.get
-    output := regex.match(p, input.body.path)
-}
-readable_post[p] := output {
-    some p in paths.read.post
-    output := regex.match(p, input.body.path)
-}
-curateable_get[p] := output {
-    some p in paths.curate.get
-    output := regex.match(p, input.body.path)
-}
-curateable_post[p] := output {
-    some p in paths.curate.post
-    output := regex.match(p, input.body.path)
+# convenience path: if a specific program is in the body, allowed = true if that program is in datasets
+allowed := true
+{
+    input.body.program in datasets
 }
-
-# which datasets can this user see for this method, path
-default datasets = []
-
-# site admins can see all programs
-datasets := all_programs
+else := true
 {
     site_admin
 }
 
-# if user is a team_member, they can access programs that allow read access for this method, path
-else := readable_programs
-{
-    valid_token
-    input.body.method = "GET"
-    regex.match(paths.read.get[_], input.body.path) == true
-}
 
-else := readable_programs
-{
-    valid_token
-    input.body.method = "POST"
-    regex.match(paths.read.post[_], input.body.path) == true
-}
+#
+# User information, for decision log
+#
 
-# if user is a site curator, they can access all programs that allow curate access for this method, path
-else := all_programs
-{
-    valid_token
-    user_key in site_roles.curator
-    input.body.method = "GET"
-    regex.match(paths.curate.get[_], input.body.path) == true
-}
+# information from the jwt
+user_key := data.idp.user_key
+issuer := data.idp.user_info.iss
 
-else := all_programs
-{
-    valid_token
-    user_key in site_roles.curator
-    input.body.method = "POST"
-    regex.match(paths.curate.post[_], input.body.path) == true
+#
+# Debugging information for decision log
+#
+
+user_is_site_admin := true {
+    user_key in data.vault.site_roles.admin
 }
+else := false
 
-else := all_programs
-{
-    valid_token
-    user_key in site_roles.curator
-    input.body.method = "DELETE"
-    regex.match(paths.curate.delete[_], input.body.path) == true
+user_is_site_curator := true {
+    user_key in data.vault.site_roles.curator
 }
+else := false
 
-# if user is a program_curator, they can access programs that allow curate access for them for this method, path
-else := curateable_programs
-{
-    valid_token
+# true if the path and method in the input match something in paths.json
+path_method_registered := true {
     input.body.method = "GET"
-    regex.match(paths.curate.get[_], input.body.path) == true
+    object.union(data.calculate.readable_get, data.calculate.curateable_get)[_]
 }
-
-else := curateable_programs
-{
-    valid_token
+else := true {
     input.body.method = "POST"
-    regex.match(paths.curate.post[_], input.body.path) == true
+    object.union(data.calculate.readable_post, data.calculate.curateable_post)[_]
 }
-
-else := curateable_programs
-{
-    valid_token
+else := true {
     input.body.method = "DELETE"
-    regex.match(paths.curate.delete[_], input.body.path) == true
+    data.calculate.curateable_delete[_]
 }
+else := false
 
-# convenience path: if a specific program is in the body, allowed = true if that program is in datasets
-allowed := true
-{
-    input.body.program in datasets
-}
-else := true
-{
-    site_admin
-}
+# programs the user is listed as a team member for
+team_member_programs := object.keys(data.calculate.team_readable_programs)
+
+# programs the user is approved by dac for
+dac_programs := object.keys(data.vault.user_programs)
 
+# programs the user is listed as a program curator for
+curator_programs := data.calculate.curateable_programs

From 68c81963a5249a950757eafe21a9ebbbc3817693 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 20 Sep 2024 18:21:10 -0700
Subject: [PATCH 32/38] Update test_opa_permissions.py

---
 tests/test_opa_permissions.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/test_opa_permissions.py b/tests/test_opa_permissions.py
index a340826..0fe5446 100644
--- a/tests/test_opa_permissions.py
+++ b/tests/test_opa_permissions.py
@@ -188,6 +188,7 @@ def evaluate_opa(user, input, key, expected_result, site_roles, users, programs)
     args = [
         "./opa", "eval",
         "--data", "permissions_engine/authz.rego",
+        "--data", "permissions_engine/calculate.rego",
         "--data", "permissions_engine/permissions.rego",
     ]
     vault = setup_vault(user, site_roles, users, programs)
@@ -217,8 +218,8 @@ def evaluate_opa(user, input, key, expected_result, site_roles, users, programs)
                 print(json.dumps({"input": input}))
                 p = subprocess.run(args, stdout=subprocess.PIPE)
                 r =  json.loads(p.stdout)
-                print(r)
                 result =r['result'][0]['expressions'][0]['value']
+                print(result)
                 if key in result:
                     assert result[key] == expected_result
                 else:

From b0d76b1fc7ac0bf3f3c8d90409466f59bab77a27 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 20 Sep 2024 22:17:57 -0700
Subject: [PATCH 33/38] mask out token from input

---
 permissions_engine/mask.rego | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 permissions_engine/mask.rego

diff --git a/permissions_engine/mask.rego b/permissions_engine/mask.rego
new file mode 100644
index 0000000..9f3b16a
--- /dev/null
+++ b/permissions_engine/mask.rego
@@ -0,0 +1,6 @@
+package system.log
+
+import rego.v1
+
+# To mask certain fields unconditionally, omit the rule body.
+mask contains "/input/token"
\ No newline at end of file

From dca262ac642113b4d911de0496a349759a29474b Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Mon, 23 Sep 2024 15:14:52 -0700
Subject: [PATCH 34/38] Update requirements.txt

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 0594cb5..75e10e2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 requests
 jq
 pytest==7.2.0
-candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.4.5
+candigv2-authx@git+https://github.com/CanDIG/candigv2-authx.git@v2.5.0
 candigv2-logging@git+https://github.com/CanDIG/candigv2-logging.git@v1.0.0

From e330cb8f894534eaf5713c9dc1bace931555d3a8 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Wed, 25 Sep 2024 15:13:53 -0700
Subject: [PATCH 35/38] authx can always see permissions

---
 permissions_engine/authz.rego | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/permissions_engine/authz.rego b/permissions_engine/authz.rego
index 91fa6c9..216be92 100644
--- a/permissions_engine/authz.rego
+++ b/permissions_engine/authz.rego
@@ -32,9 +32,7 @@ allow {
 
 # The authx library uses these paths:
 authx_paths = {
-    "datasets": ["v1", "data", "permissions", "datasets"],
-    "allowed": ["v1", "data", "permissions", "allowed"],
-    "site_admin": ["v1", "data", "permissions", "site_admin"],
+    "permissions": ["v1", "data", "permissions"],
     "user_id": ["v1", "data", "idp", "user_key"]
 }
 

From e2287cef452bba90f1c1ab21bc1d047537967322 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 3 Oct 2024 13:48:20 -0700
Subject: [PATCH 36/38] Clarify curate paths for site curators

---
 defaults/paths.json | 51 +++++++++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/defaults/paths.json b/defaults/paths.json
index 171ac47..7163c9d 100644
--- a/defaults/paths.json
+++ b/defaults/paths.json
@@ -2,16 +2,16 @@
     "paths": {
         "read": {
             "get": [
-              "/v3/discovery/?.*",
-              "/v3/authorized/?.*",
-              "/htsget/v1/variants/?.*",
-              "/htsget/v1/variants/search",
-              "/htsget/v1/reads/?.*",
-              "/htsget/v1/samples/?.*",
-              "/ga4gh/drs/v1/objects/?.*",
-              "/ga4gh/drs/v1/cohorts/?.*",
-              "/ga4gh/drs/v1/cohorts/?.*/status",
-              "/beacon/v2/g_variants/?.*"
+                "/v3/discovery/?.*",
+                "/v3/authorized/?.*",
+                "/htsget/v1/variants/?.*",
+                "/htsget/v1/variants/search",
+                "/htsget/v1/reads/?.*",
+                "/htsget/v1/samples/?.*",
+                "/ga4gh/drs/v1/objects/?.*",
+                "/ga4gh/drs/v1/cohorts/?.*",
+                "/ga4gh/drs/v1/cohorts/?.*/status",
+                "/beacon/v2/g_variants/?.*"
             ],
             "post": [
                 "/htsget/v1/variants/search",
@@ -20,14 +20,29 @@
             ]
         },
         "curate": {
-          "get": [
-            "/htsget/v1/variants/?.*/index",
-            "/htsget/v1/variants/?.*/verify",
-            "/htsget/v1/reads/?.*/index",
-            "/htsget/v1/reads/?.*/verify"
-          ],
-          "post": ["/ingest/?.*", "/ga4gh/drs/v1/?.*", "/v3/ingest/?.*"],
-          "delete": ["/ingest/?.*", "/ga4gh/drs/v1/?.*", "/v3/ingest/?.*"]
+            "get": [
+                "/ingest/?.*",
+                "/htsget/v1/variants/?.*/index",
+                "/htsget/v1/variants/?.*/verify",
+                "/htsget/v1/reads/?.*/index",
+                "/htsget/v1/reads/?.*/verify"
+            ],
+            "post": [
+                "/ingest/s3-credential/?.*",
+                "/ingest/program/?.*",
+                "/ingest/user/?.*",
+                "/ingest/genomic",
+                "/ingest/clinical",
+                "/ga4gh/drs/v1/?.*",
+                "/v3/ingest/?.*"
+            ],
+            "delete": [
+                "/ingest/s3-credential/?.*",
+                "/ingest/program/?.*",
+                "/ingest/user/?.*",
+                "/ga4gh/drs/v1/?.*",
+                "/v3/ingest/?.*"
+            ]
         }
     }
 }
\ No newline at end of file

From 04df88f044bf02f27d415b041903f535810d7f65 Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Thu, 3 Oct 2024 14:36:14 -0700
Subject: [PATCH 37/38] site curator is allowed to curate always

---
 permissions_engine/calculate.rego   |  4 +++
 permissions_engine/permissions.rego | 45 +++++++++++++++++++----------
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/permissions_engine/calculate.rego b/permissions_engine/calculate.rego
index 4077de1..a87b109 100644
--- a/permissions_engine/calculate.rego
+++ b/permissions_engine/calculate.rego
@@ -23,6 +23,10 @@ site_admin = true {
     user_key in site_roles.admin
 }
 
+site_curator = true {
+    user_key in site_roles.curator
+}
+
 #
 # what programs are available to this user?
 #
diff --git a/permissions_engine/permissions.rego b/permissions_engine/permissions.rego
index c3aff99..b877457 100644
--- a/permissions_engine/permissions.rego
+++ b/permissions_engine/permissions.rego
@@ -13,12 +13,35 @@ site_admin := data.calculate.site_admin {
     valid_token
 }
 
+site_curator := data.calculate.site_curator {
+    valid_token
+}
+
 datasets := data.calculate.datasets {
     valid_token
 }
 else := []
 
-# convenience path: if a specific program is in the body, allowed = true if that program is in datasets
+
+# true if the path and method in the input match something in paths.json
+path_method_registered := true {
+    input.body.method = "GET"
+    object.union(data.calculate.readable_get, data.calculate.curateable_get)[_]
+}
+else := true {
+    input.body.method = "POST"
+    object.union(data.calculate.readable_post, data.calculate.curateable_post)[_]
+}
+else := true {
+    input.body.method = "DELETE"
+    data.calculate.curateable_delete[_]
+}
+else := false
+
+
+# if a specific program is in the body, allowed = true if that program is in datasets
+# or if the user is a site admin
+# or if the user is a site curator and wants to curate something
 allowed := true
 {
     input.body.program in datasets
@@ -27,6 +50,11 @@ else := true
 {
     site_admin
 }
+else := true
+{
+    site_curator
+    path_method_registered
+}
 
 
 #
@@ -51,21 +79,6 @@ user_is_site_curator := true {
 }
 else := false
 
-# true if the path and method in the input match something in paths.json
-path_method_registered := true {
-    input.body.method = "GET"
-    object.union(data.calculate.readable_get, data.calculate.curateable_get)[_]
-}
-else := true {
-    input.body.method = "POST"
-    object.union(data.calculate.readable_post, data.calculate.curateable_post)[_]
-}
-else := true {
-    input.body.method = "DELETE"
-    data.calculate.curateable_delete[_]
-}
-else := false
-
 # programs the user is listed as a team member for
 team_member_programs := object.keys(data.calculate.team_readable_programs)
 

From 9fbb78b67c2a59b43ee82ed54215b45845afebad Mon Sep 17 00:00:00 2001
From: Daisie Huang <daisieh@gmail.com>
Date: Fri, 4 Oct 2024 14:38:01 -0700
Subject: [PATCH 38/38] redo datasets calculation

---
 permissions_engine/calculate.rego | 40 ++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/permissions_engine/calculate.rego b/permissions_engine/calculate.rego
index a87b109..5972d73 100644
--- a/permissions_engine/calculate.rego
+++ b/permissions_engine/calculate.rego
@@ -92,39 +92,61 @@ datasets := all_programs
     site_admin
 }
 
-# if user is a team_member, they can access programs that allow read access for this method, path
-else := readable_programs
+# if user is a site curator, they can access all programs that allow curate access for this method, path
+else := all_programs
 {
+    user_key in site_roles.curator
     input.body.method = "GET"
-    regex.match(paths.read.get[_], input.body.path) == true
+    regex.match(paths.curate.get[_], input.body.path) == true
 }
 
-else := readable_programs
+else := all_programs
 {
+    user_key in site_roles.curator
     input.body.method = "POST"
-    regex.match(paths.read.post[_], input.body.path) == true
+    regex.match(paths.curate.post[_], input.body.path) == true
 }
 
-# if user is a site curator, they can access all programs that allow curate access for this method, path
+else := all_programs
+{
+    user_key in site_roles.curator
+    input.body.method = "DELETE"
+    regex.match(paths.curate.delete[_], input.body.path) == true
+}
+
+# if user is a site curator, they can access all programs that allow read access for this method, path
 else := all_programs
 {
     user_key in site_roles.curator
     input.body.method = "GET"
-    regex.match(paths.curate.get[_], input.body.path) == true
+    regex.match(paths.read.get[_], input.body.path) == true
 }
 
 else := all_programs
 {
     user_key in site_roles.curator
     input.body.method = "POST"
-    regex.match(paths.curate.post[_], input.body.path) == true
+    regex.match(paths.read.post[_], input.body.path) == true
 }
 
 else := all_programs
 {
     user_key in site_roles.curator
     input.body.method = "DELETE"
-    regex.match(paths.curate.delete[_], input.body.path) == true
+    regex.match(paths.read.delete[_], input.body.path) == true
+}
+
+# if user is a team_member, they can access programs that allow read access for this method, path
+else := readable_programs
+{
+    input.body.method = "GET"
+    regex.match(paths.read.get[_], input.body.path) == true
+}
+
+else := readable_programs
+{
+    input.body.method = "POST"
+    regex.match(paths.read.post[_], input.body.path) == true
 }
 
 # if user is a program_curator, they can access programs that allow curate access for them for this method, path