From af30748f2b4c273c92639f75619f649b4253e4c6 Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Tue, 21 Nov 2023 22:52:53 +0000 Subject: [PATCH 1/5] [DM-41830] First try to get a new Siav2 service auth working This will probably get reorganized. --- src/lsst/rsp/catalog.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/lsst/rsp/catalog.py b/src/lsst/rsp/catalog.py index 3e7e390..b592d44 100644 --- a/src/lsst/rsp/catalog.py +++ b/src/lsst/rsp/catalog.py @@ -10,6 +10,18 @@ from .utils import get_access_token +def _get_cutout_url() -> str: + return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/cutout" + + +def _get_datalink_url() -> str: + return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/datalink" + + +def _get_siav2_url() -> str: + return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/siav2" + + def _get_tap_url(env_var: str, url: str) -> str: tapurl = os.getenv("EXTERNAL_" + env_var + "_URL") if not tapurl: @@ -19,14 +31,6 @@ def _get_tap_url(env_var: str, url: str) -> str: return tapurl -def _get_datalink_url() -> str: - return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/datalink" - - -def _get_cutout_url() -> str: - return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/cutout" - - def _get_auth() -> Optional[pyvo.auth.authsession.AuthSession]: tap_url = _get_tap_url("TAP", "tap") obstap_url = _get_tap_url("OBSTAP", "obstap") @@ -38,8 +42,10 @@ def _get_auth() -> Optional[pyvo.auth.authsession.AuthSession]: s.headers["Authorization"] = "Bearer " + tok auth = pyvo.auth.authsession.AuthSession() auth.credentials.set("lsst-token", s) - auth.add_security_method_for_url(_get_datalink_url(), "lsst-token") auth.add_security_method_for_url(_get_cutout_url(), "lsst-token") + auth.add_security_method_for_url(_get_datalink_url(), "lsst-token") + auth.add_security_method_for_url(_get_siav2_url(), "lsst-token") + auth.add_security_method_for_url(_get_siav2_url() + "/query", "lsst-token") auth.add_security_method_for_url(tap_url, "lsst-token") auth.add_security_method_for_url(tap_url + "/sync", "lsst-token") auth.add_security_method_for_url(tap_url + "/async", "lsst-token") @@ -52,6 +58,7 @@ def _get_auth() -> Optional[pyvo.auth.authsession.AuthSession]: auth.add_security_method_for_url(ssotap_url + "/sync", "lsst-token") auth.add_security_method_for_url(ssotap_url + "/async", "lsst-token") auth.add_security_method_for_url(ssotap_url + "/tables", "lsst-token") + return auth @@ -65,6 +72,10 @@ def get_obstap_service() -> pyvo.dal.TAPService: return get_tap_service("obstap") +def get_siav2_service(*args: str) -> pyvo.dal.SIA2Service: + return SIA2Service(_get_siav2_url(), _get_auth()) + + def get_tap_service(*args: str) -> pyvo.dal.TAPService: if len(args) == 0: warnings.warn( From 4b28f858d8851b3da324efc35e8f5ed3a09c29a6 Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Wed, 29 Nov 2023 15:53:17 -0700 Subject: [PATCH 2/5] [DM-41830] Major refactor Pull the auth parts into utils along with other utility functions Make catalog.py a lot shorter and break that off from siav2 Make a new service.py to hold the SIAv2 service --- src/lsst/rsp/catalog.py | 72 +++-------------------------------------- src/lsst/rsp/service.py | 12 +++++++ src/lsst/rsp/utils.py | 48 +++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 67 deletions(-) create mode 100644 src/lsst/rsp/service.py diff --git a/src/lsst/rsp/catalog.py b/src/lsst/rsp/catalog.py index b592d44..13de5d4 100644 --- a/src/lsst/rsp/catalog.py +++ b/src/lsst/rsp/catalog.py @@ -3,63 +3,9 @@ from typing import Optional import pyvo -import pyvo.auth.authsession -import requests from deprecated import deprecated -from .utils import get_access_token - - -def _get_cutout_url() -> str: - return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/cutout" - - -def _get_datalink_url() -> str: - return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/datalink" - - -def _get_siav2_url() -> str: - return os.getenv("EXTERNAL_INSTANCE_URL", "") + "/api/siav2" - - -def _get_tap_url(env_var: str, url: str) -> str: - tapurl = os.getenv("EXTERNAL_" + env_var + "_URL") - if not tapurl: - tapurl = (os.getenv("EXTERNAL_INSTANCE_URL") or "") + ( - os.getenv(env_var + "_ROUTE") or "/api/" + url - ) - return tapurl - - -def _get_auth() -> Optional[pyvo.auth.authsession.AuthSession]: - tap_url = _get_tap_url("TAP", "tap") - obstap_url = _get_tap_url("OBSTAP", "obstap") - ssotap_url = _get_tap_url("SSOTAP", "ssotap") - s = requests.Session() - tok = get_access_token() - if not tok: - return None - s.headers["Authorization"] = "Bearer " + tok - auth = pyvo.auth.authsession.AuthSession() - auth.credentials.set("lsst-token", s) - auth.add_security_method_for_url(_get_cutout_url(), "lsst-token") - auth.add_security_method_for_url(_get_datalink_url(), "lsst-token") - auth.add_security_method_for_url(_get_siav2_url(), "lsst-token") - auth.add_security_method_for_url(_get_siav2_url() + "/query", "lsst-token") - auth.add_security_method_for_url(tap_url, "lsst-token") - auth.add_security_method_for_url(tap_url + "/sync", "lsst-token") - auth.add_security_method_for_url(tap_url + "/async", "lsst-token") - auth.add_security_method_for_url(tap_url + "/tables", "lsst-token") - auth.add_security_method_for_url(obstap_url, "lsst-token") - auth.add_security_method_for_url(obstap_url + "/sync", "lsst-token") - auth.add_security_method_for_url(obstap_url + "/async", "lsst-token") - auth.add_security_method_for_url(obstap_url + "/tables", "lsst-token") - auth.add_security_method_for_url(ssotap_url, "lsst-token") - auth.add_security_method_for_url(ssotap_url + "/sync", "lsst-token") - auth.add_security_method_for_url(ssotap_url + "/async", "lsst-token") - auth.add_security_method_for_url(ssotap_url + "/tables", "lsst-token") - - return auth +from .utils import get_pyvo_auth, get_service_url @deprecated(reason='Please use get_tap_service("tap")') @@ -72,10 +18,6 @@ def get_obstap_service() -> pyvo.dal.TAPService: return get_tap_service("obstap") -def get_siav2_service(*args: str) -> pyvo.dal.SIA2Service: - return SIA2Service(_get_siav2_url(), _get_auth()) - - def get_tap_service(*args: str) -> pyvo.dal.TAPService: if len(args) == 0: warnings.warn( @@ -87,12 +29,8 @@ def get_tap_service(*args: str) -> pyvo.dal.TAPService: else: database = args[0] - if database == "tap": - tap_url = _get_tap_url("TAP", "tap") - elif database == "obstap": - tap_url = _get_tap_url("OBSTAP", "obstap") - elif database == "ssotap": - tap_url = _get_tap_url("SSOTAP", "ssotap") + if database in ["tap", "obstap", "ssotap"]: + tap_url = get_service_url(database) else: raise Exception(database + " is not a valid tap service") @@ -106,7 +44,7 @@ def get_tap_service(*args: str) -> pyvo.dal.TAPService: # with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - ts = pyvo.dal.TAPService(tap_url, _get_auth()) + ts = pyvo.dal.TAPService(tap_url, get_pyvo_auth()) return ts @@ -121,5 +59,5 @@ def retrieve_query(query_url: str) -> pyvo.dal.AsyncTAPJob: # with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - atj = pyvo.dal.AsyncTAPJob(query_url, _get_auth()) + atj = pyvo.dal.AsyncTAPJob(query_url, get_pyvo_auth()) return atj diff --git a/src/lsst/rsp/service.py b/src/lsst/rsp/service.py new file mode 100644 index 0000000..348b6c2 --- /dev/null +++ b/src/lsst/rsp/service.py @@ -0,0 +1,12 @@ +import pyvo +from pyvo.dal import SIA2Service + +from .utils import get_pyvo_auth, get_service_url + +def get_siav2_service(*args: str) -> pyvo.dal.SIA2Service: + if len(args) == 0: + ds = "dp0.2" + elif args == "latiss": + ds = "latiss" + + return SIA2Service(get_service_url("siav2"), get_pyvo_auth()) diff --git a/src/lsst/rsp/utils.py b/src/lsst/rsp/utils.py index 3ffca45..591300e 100644 --- a/src/lsst/rsp/utils.py +++ b/src/lsst/rsp/utils.py @@ -6,6 +6,9 @@ from typing import Any, Optional, Union import bokeh.io +import pyvo.auth.authsession +import requests + _NO_K8S = False @@ -53,6 +56,51 @@ def get_hostname() -> str: return os.environ.get("HOSTNAME") or "localhost" +def get_service_url(name: str, env_name=None) -> str: + if not env_name: + env_name = name.upper() + + url = os.getenv(f"EXTERNAL_{env_name}_URL") + if url: + return url + + fqdn = os.getenv("EXTERNAL_INSTANCE_URL") or "" + path = os.getenv(f"{env_name}_ROUTE") or f"/api/{name}" + return f"{fqdn}/{path}" + + +def get_pyvo_auth() -> Optional[pyvo.auth.authsession.AuthSession]: + """Utility function to create a pyvo compatible auth object.""" + tap_url = get_service_url("tap") + obstap_url = get_service_url("obstap") + ssotap_url = get_service_url("ssotap") + siav2_url = get_service_url("siav2") + s = requests.Session() + tok = get_access_token() + if not tok: + return None + s.headers["Authorization"] = "Bearer " + tok + auth = pyvo.auth.authsession.AuthSession() + auth.credentials.set("lsst-token", s) + auth.add_security_method_for_url(get_service_url("cutout"), "lsst-token") + auth.add_security_method_for_url(get_service_url("datalink"), "lsst-token") + auth.add_security_method_for_url(siav2_url, "lsst-token") + auth.add_security_method_for_url(siav2_url + "/query", "lsst-token") + auth.add_security_method_for_url(tap_url, "lsst-token") + auth.add_security_method_for_url(tap_url + "/sync", "lsst-token") + auth.add_security_method_for_url(tap_url + "/async", "lsst-token") + auth.add_security_method_for_url(tap_url + "/tables", "lsst-token") + auth.add_security_method_for_url(obstap_url, "lsst-token") + auth.add_security_method_for_url(obstap_url + "/sync", "lsst-token") + auth.add_security_method_for_url(obstap_url + "/async", "lsst-token") + auth.add_security_method_for_url(obstap_url + "/tables", "lsst-token") + auth.add_security_method_for_url(ssotap_url, "lsst-token") + auth.add_security_method_for_url(ssotap_url + "/sync", "lsst-token") + auth.add_security_method_for_url(ssotap_url + "/async", "lsst-token") + auth.add_security_method_for_url(ssotap_url + "/tables", "lsst-token") + return auth + + def show_with_bokeh_server(obj: Any) -> None: """Method to wrap bokeh with proxy URL""" From ca0ea29e49397fa180fbe2e866e14c3f2e36367a Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Wed, 29 Nov 2023 16:01:42 -0700 Subject: [PATCH 3/5] [DM-41830] Make up for obstap -> live --- src/lsst/rsp/catalog.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lsst/rsp/catalog.py b/src/lsst/rsp/catalog.py index 13de5d4..eaddd97 100644 --- a/src/lsst/rsp/catalog.py +++ b/src/lsst/rsp/catalog.py @@ -13,12 +13,14 @@ def get_catalog() -> pyvo.dal.TAPService: return get_tap_service("tap") -@deprecated(reason='Please use get_tap_service("obstap")') +@deprecated(reason='Please use get_tap_service("live")') def get_obstap_service() -> pyvo.dal.TAPService: - return get_tap_service("obstap") + return get_tap_service("live") def get_tap_service(*args: str) -> pyvo.dal.TAPService: + """Returns a TAP service instance to interact with the + requested TAP service.""" if len(args) == 0: warnings.warn( 'get_tap_service() is deprecated, use get_tap_service("tap")', @@ -29,7 +31,12 @@ def get_tap_service(*args: str) -> pyvo.dal.TAPService: else: database = args[0] - if database in ["tap", "obstap", "ssotap"]: + # We renamed the name of the TAP service from obstap + # to live + if database == "obstap": + database = "live" + + if database in ["live", "tap", "ssotap"]: tap_url = get_service_url(database) else: raise Exception(database + " is not a valid tap service") From 37297bcd8e08b5bdbcd7286a3e2297b64bc53d2d Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Wed, 29 Nov 2023 16:41:06 -0700 Subject: [PATCH 4/5] [DM-41830] Add some sanity checking on siav2 labels --- src/lsst/rsp/service.py | 43 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/lsst/rsp/service.py b/src/lsst/rsp/service.py index 348b6c2..e86efb3 100644 --- a/src/lsst/rsp/service.py +++ b/src/lsst/rsp/service.py @@ -1,12 +1,47 @@ +import os + import pyvo from pyvo.dal import SIA2Service from .utils import get_pyvo_auth, get_service_url -def get_siav2_service(*args: str) -> pyvo.dal.SIA2Service: + +LSST_CLOUD = [ + "https://data.lsst.cloud", + "https://data-int.lsst.cloud", + "https://data-dev.lsst.cloud" +] + +USDF = [ + "https://usdf-rsp.slac.stanford.edu/", + "https://usdf-rsp-dev.slac.stanford.edu/" +] + + +def get_siav2_service(*args: str) -> SIA2Service: + """Return a configured SIA2Service object that is ready to use.""" + fqdn = os.environ["EXTERNAL_INSTANCE_URL"] + if len(args) == 0: - ds = "dp0.2" - elif args == "latiss": - ds = "latiss" + # Use the default for each environment + if fqdn in LSST_CLOUD: + label = "dp0.2" + elif fqdn in USDF: + label = "latiss" + else: + raise Exception("Unknown environment" + fqdn) + else: + label = args[0] + + # If a label is passed, check that. + if label not in ["dp0.2", "latiss"]: + raise Exception(label + " is not a valid siav2 label") + + if label == "latiss" and fqdn not in USDF: + raise Exception(label + " data not available at your location") + if label == "dp0.2" and fqdn not in LSST_CLOUD: + raise Exception(label + " data not available at your location") + # No matter what, we've only got one sia server per environment + # so for now just do some checking. return SIA2Service(get_service_url("siav2"), get_pyvo_auth()) From 83f6a7a306b6bdee981b7fcc7b970decb0ebde21 Mon Sep 17 00:00:00 2001 From: Christine Banek Date: Mon, 4 Dec 2023 13:50:26 -0700 Subject: [PATCH 5/5] [DM-41830] Go to a simpler refactor about the labels --- setup.cfg | 2 +- src/lsst/rsp/catalog.py | 2 -- src/lsst/rsp/service.py | 38 ++------------------------------------ src/lsst/rsp/utils.py | 8 ++------ 4 files changed, 5 insertions(+), 45 deletions(-) diff --git a/setup.cfg b/setup.cfg index 77cd1dd..e002362 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,7 @@ install_requires = kubernetes maproxy notebook - pyvo + pyvo @ git+https://github.com/astropy/pyvo.git@7682a1809cb2169cc2ffd9313feb6a4abd7f7e37 requests tornado diff --git a/src/lsst/rsp/catalog.py b/src/lsst/rsp/catalog.py index eaddd97..6e11b8c 100644 --- a/src/lsst/rsp/catalog.py +++ b/src/lsst/rsp/catalog.py @@ -1,6 +1,4 @@ -import os import warnings -from typing import Optional import pyvo from deprecated import deprecated diff --git a/src/lsst/rsp/service.py b/src/lsst/rsp/service.py index e86efb3..c194a7d 100644 --- a/src/lsst/rsp/service.py +++ b/src/lsst/rsp/service.py @@ -1,45 +1,11 @@ -import os - -import pyvo from pyvo.dal import SIA2Service from .utils import get_pyvo_auth, get_service_url -LSST_CLOUD = [ - "https://data.lsst.cloud", - "https://data-int.lsst.cloud", - "https://data-dev.lsst.cloud" -] - -USDF = [ - "https://usdf-rsp.slac.stanford.edu/", - "https://usdf-rsp-dev.slac.stanford.edu/" -] - - -def get_siav2_service(*args: str) -> SIA2Service: +def get_siav2_service(label: str) -> SIA2Service: """Return a configured SIA2Service object that is ready to use.""" - fqdn = os.environ["EXTERNAL_INSTANCE_URL"] - - if len(args) == 0: - # Use the default for each environment - if fqdn in LSST_CLOUD: - label = "dp0.2" - elif fqdn in USDF: - label = "latiss" - else: - raise Exception("Unknown environment" + fqdn) - else: - label = args[0] - - # If a label is passed, check that. - if label not in ["dp0.2", "latiss"]: - raise Exception(label + " is not a valid siav2 label") - - if label == "latiss" and fqdn not in USDF: - raise Exception(label + " data not available at your location") - if label == "dp0.2" and fqdn not in LSST_CLOUD: + if label != "staff": raise Exception(label + " data not available at your location") # No matter what, we've only got one sia server per environment diff --git a/src/lsst/rsp/utils.py b/src/lsst/rsp/utils.py index 591300e..a2a223b 100644 --- a/src/lsst/rsp/utils.py +++ b/src/lsst/rsp/utils.py @@ -9,7 +9,6 @@ import pyvo.auth.authsession import requests - _NO_K8S = False try: @@ -56,7 +55,7 @@ def get_hostname() -> str: return os.environ.get("HOSTNAME") or "localhost" -def get_service_url(name: str, env_name=None) -> str: +def get_service_url(name: str, env_name: Optional[str] = None) -> str: if not env_name: env_name = name.upper() @@ -129,10 +128,7 @@ def jupyter_proxy_url(port: Optional[int] = None) -> str: full_url = urllib.parse.urljoin(user_url, proxy_url_path) return full_url - # The type: ignore is needed because the alternate form of notebook_url - # (https://docs.bokeh.org/en/latest/docs/reference/io.html#bokeh.io.show) - # isn't in bokeh's type hints. - bokeh.io.show(obj=obj, notebook_url=jupyter_proxy_url) # type:ignore + bokeh.io.show(obj=obj, notebook_url=jupyter_proxy_url) def get_pod() -> Optional[client.V1Pod]: