From 4c3e555fb26cb7122d5fb09f874035b34e13171c Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 17:39:43 -0400 Subject: [PATCH 01/11] temp: Log all image references This is to help find the naming formula for image references. --- src/noteburst/jupyterclient/jupyterlab.py | 3 ++- src/noteburst/jupyterclient/labcontroller.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/noteburst/jupyterclient/jupyterlab.py b/src/noteburst/jupyterclient/jupyterlab.py index 8399b5a..f74f903 100644 --- a/src/noteburst/jupyterclient/jupyterlab.py +++ b/src/noteburst/jupyterclient/jupyterlab.py @@ -16,7 +16,7 @@ import websockets from httpx import Cookies, Timeout from pydantic import BaseModel, Field -from structlog import BoundLogger +from structlog.stdlib import BoundLogger from websockets.client import WebSocketClientProtocol from websockets.exceptions import WebSocketException from websockets.typing import Data as WebsocketData @@ -442,6 +442,7 @@ def lab_controller(self) -> LabControllerClient: http_client=self.http_client, token=noteburst_config.gafaelfawr_token.get_secret_value(), url_prefix=noteburst_config.nublado_controller_path_prefix, + logger=self.logger, ) return self._lab_controller_client diff --git a/src/noteburst/jupyterclient/labcontroller.py b/src/noteburst/jupyterclient/labcontroller.py index 79365a8..e99d6cd 100644 --- a/src/noteburst/jupyterclient/labcontroller.py +++ b/src/noteburst/jupyterclient/labcontroller.py @@ -7,6 +7,7 @@ import httpx from pydantic import BaseModel, ConfigDict, Field +from structlog.stdlib import BoundLogger from noteburst.config import config @@ -95,7 +96,9 @@ class LabControllerImages(BaseModel): ) """Pydantic model configuration.""" - def get_by_reference(self, reference: str) -> JupyterImage | None: + def get_by_reference( + self, reference: str, logger: BoundLogger + ) -> JupyterImage | None: """Get the JupyterImage with a corresponding reference. Parameters @@ -109,6 +112,7 @@ def get_by_reference(self, reference: str) -> JupyterImage | None: Returns the JupyterImage if found, None otherwise. """ for image in self.all: + logger.info("Image reference", image=image.reference) if reference == image.reference: return image @@ -142,10 +146,12 @@ def __init__( http_client: httpx.AsyncClient, token: str, url_prefix: str, + logger: BoundLogger, ) -> None: self._http_client = http_client self._token = token self._url_prefix = url_prefix + self._logger = logger async def get_latest_weekly(self) -> JupyterImage: """Image for the latest weekly version. @@ -200,7 +206,7 @@ async def get_by_reference(self, reference: str) -> JupyterImage: An error occurred talking to JupyterLab Controller. """ images = await self._get_images() - image = images.get_by_reference(reference) + image = images.get_by_reference(reference, logger=self._logger) if image is None: raise LabControllerError( f"No image with reference {reference} found." From cb067157969a0cf4fc1375fa7a485fe2748ff605 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 17:52:46 -0400 Subject: [PATCH 02/11] temp: revert back to user token_type --- src/noteburst/jupyterclient/user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/noteburst/jupyterclient/user.py b/src/noteburst/jupyterclient/user.py index 51bc8f2..03a4bd7 100644 --- a/src/noteburst/jupyterclient/user.py +++ b/src/noteburst/jupyterclient/user.py @@ -97,7 +97,7 @@ async def create( token_request_data = { "username": username, "name": "Noteburst", - "token_type": "service", + "token_type": "user", "token_name": f"noteburst {float(time.time())!s}", "scopes": scopes, "expires": int(time.time() + lifetime), From 0f284acb59832ae0c084d4a5a3f59eafd0ea21c0 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 17:57:27 -0400 Subject: [PATCH 03/11] fixup --- tests/support/gafaelfawr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/gafaelfawr.py b/tests/support/gafaelfawr.py index 2a31b17..d7b4ced 100644 --- a/tests/support/gafaelfawr.py +++ b/tests/support/gafaelfawr.py @@ -55,7 +55,7 @@ def handler(request: httpx.Request) -> httpx.Response: # so we can't check it here. assert request_json == { "username": ANY, - "token_type": "service", + "token_type": "user", "token_name": ANY, "scopes": ["exec:notebook"], "expires": ANY, From c40f8f12beb5dd371c59ba309e93de54e72f7f43 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:11:34 -0400 Subject: [PATCH 04/11] Log the error body from service token --- src/noteburst/jupyterclient/user.py | 3 ++- tests/support/gafaelfawr.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/noteburst/jupyterclient/user.py b/src/noteburst/jupyterclient/user.py index 03a4bd7..e349087 100644 --- a/src/noteburst/jupyterclient/user.py +++ b/src/noteburst/jupyterclient/user.py @@ -97,7 +97,7 @@ async def create( token_request_data = { "username": username, "name": "Noteburst", - "token_type": "user", + "token_type": "service", "token_name": f"noteburst {float(time.time())!s}", "scopes": scopes, "expires": int(time.time() + lifetime), @@ -115,6 +115,7 @@ async def create( }, json=token_request_data, ) + print(r.json()) # noqa: T201 r.raise_for_status() body = r.json() return cls( diff --git a/tests/support/gafaelfawr.py b/tests/support/gafaelfawr.py index d7b4ced..2a31b17 100644 --- a/tests/support/gafaelfawr.py +++ b/tests/support/gafaelfawr.py @@ -55,7 +55,7 @@ def handler(request: httpx.Request) -> httpx.Response: # so we can't check it here. assert request_json == { "username": ANY, - "token_type": "user", + "token_type": "service", "token_name": ANY, "scopes": ["exec:notebook"], "expires": ANY, From 3df9d545017a4349b44006344bcea3d6eb44e6b1 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:18:53 -0400 Subject: [PATCH 05/11] Log the error message from logging into JupyterHub --- src/noteburst/jupyterclient/user.py | 1 - src/noteburst/worker/main.py | 8 +++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/noteburst/jupyterclient/user.py b/src/noteburst/jupyterclient/user.py index e349087..51bc8f2 100644 --- a/src/noteburst/jupyterclient/user.py +++ b/src/noteburst/jupyterclient/user.py @@ -115,7 +115,6 @@ async def create( }, json=token_request_data, ) - print(r.json()) # noqa: T201 r.raise_for_status() body = r.json() return cls( diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index 1f9e12a..9f695e3 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -83,7 +83,13 @@ async def startup(ctx: dict[Any, Any]) -> None: jupyter_client = JupyterClient( user=authed_user, logger=logger, config=jupyter_config ) - await jupyter_client.log_into_hub() + try: + await jupyter_client.log_into_hub() + except httpx.HTTPStatusError as e: + logger.exception( + "Error logging into JupyterHub", + body=e.response.json(), + ) try: image_info = await jupyter_client.spawn_lab() logger = logger.bind(image_ref=image_info.reference) From 8235ce822f0ec13eb277aef4adf2df90a39248b2 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:19:26 -0400 Subject: [PATCH 06/11] fixup --- src/noteburst/worker/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index 9f695e3..717067b 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -90,6 +90,7 @@ async def startup(ctx: dict[Any, Any]) -> None: "Error logging into JupyterHub", body=e.response.json(), ) + raise try: image_info = await jupyter_client.spawn_lab() logger = logger.bind(image_ref=image_info.reference) From eda1609b825b15d14986718a8b84c7dbb498cb27 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:25:14 -0400 Subject: [PATCH 07/11] Test generic exception logging --- src/noteburst/worker/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index 717067b..a9b4b3a 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -91,6 +91,9 @@ async def startup(ctx: dict[Any, Any]) -> None: body=e.response.json(), ) raise + except Exception: + logger.exception("Generic error logging into JupyterHub") + raise try: image_info = await jupyter_client.spawn_lab() logger = logger.bind(image_ref=image_info.reference) From 48e35d3dd599716a7113ed9acabfd9bcfe35b433 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:30:31 -0400 Subject: [PATCH 08/11] Log the correct place --- pyproject.toml | 3 +++ src/noteburst/worker/main.py | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c75d9a0..97b3c09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,6 +176,9 @@ select = ["ALL"] "SLF001", # tests are allowed to access private members "T201", # tests are allowed to use print ] +"src/noteburst/worker/main.py" = [ + "PLR0915", # along a long function +] [tool.ruff.lint.isort] known-first-party = ["noteburst", "tests"] diff --git a/src/noteburst/worker/main.py b/src/noteburst/worker/main.py index a9b4b3a..3d73d2a 100644 --- a/src/noteburst/worker/main.py +++ b/src/noteburst/worker/main.py @@ -73,11 +73,19 @@ async def startup(ctx: dict[Any, Any]) -> None: user = User( username=identity.username, uid=identity.uid, gid=identity.gid ) - authed_user = await user.login( - scopes=config.parsed_worker_token_scopes, - http_client=http_client, - token_lifetime=config.worker_token_lifetime, - ) + try: + authed_user = await user.login( + scopes=config.parsed_worker_token_scopes, + http_client=http_client, + token_lifetime=config.worker_token_lifetime, + ) + except httpx.HTTPStatusError as e: + logger.exception( + "Error authenticating the worker's user", + body=e.response.json(), + status_code=e.response.status_code, + ) + raise logger.info("Authenticated the worker's user.") jupyter_client = JupyterClient( From 680039b05b2155afc1a4576861d9f6fcfeaab036 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:36:27 -0400 Subject: [PATCH 09/11] Drop token_name field service user tokens can't have names. --- src/noteburst/jupyterclient/user.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/noteburst/jupyterclient/user.py b/src/noteburst/jupyterclient/user.py index 51bc8f2..23737ed 100644 --- a/src/noteburst/jupyterclient/user.py +++ b/src/noteburst/jupyterclient/user.py @@ -98,7 +98,6 @@ async def create( "username": username, "name": "Noteburst", "token_type": "service", - "token_name": f"noteburst {float(time.time())!s}", "scopes": scopes, "expires": int(time.time() + lifetime), } From 537e109b197fb31d41a8a22c9df1a107d6bb2512 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:38:02 -0400 Subject: [PATCH 10/11] fixup --- tests/support/gafaelfawr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/support/gafaelfawr.py b/tests/support/gafaelfawr.py index 2a31b17..3797927 100644 --- a/tests/support/gafaelfawr.py +++ b/tests/support/gafaelfawr.py @@ -56,7 +56,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert request_json == { "username": ANY, "token_type": "service", - "token_name": ANY, "scopes": ["exec:notebook"], "expires": ANY, "name": "Noteburst", From 19f83f66e69fe70bc5ba2edd935300a7c09f0714 Mon Sep 17 00:00:00 2001 From: Jonathan Sick Date: Fri, 3 May 2024 18:39:42 -0400 Subject: [PATCH 11/11] fixup --- tests/support/gafaelfawr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/support/gafaelfawr.py b/tests/support/gafaelfawr.py index 3797927..2d854bd 100644 --- a/tests/support/gafaelfawr.py +++ b/tests/support/gafaelfawr.py @@ -68,7 +68,6 @@ def handler(request: httpx.Request) -> httpx.Response: assert request_json["uid"] == uid if gid: assert request_json["gid"] == gid - assert request_json["token_name"].startswith("noteburst ") assert request_json["expires"] > time.time() response = {"token": make_gafaelfawr_token(request_json["username"])} return httpx.Response(200, json=response, request=request)