DINO-1139: add LASS resiliency

internal/leetcode-auth-stealer-service/src/grpcserve/service/auth.py

@@ -1,4 +1,5 @@
 import asyncio
+from collections.abc import Callable
 
 import grpc
 import structlog
@@ -8,7 +9,7 @@
     LeetcodeAuthStealerService_pb2,
     LeetcodeAuthStealerService_pb2_grpc,
 )
-from prom import record_auth_failure, record_auth_success
+from prom import AuthFailureReason, record_auth_failure, record_auth_success
 from provider.camou import CamouProvider
 from provider.dto import (
     AuthenticationTimeout,
@@ -24,9 +25,23 @@
 class AuthService(
     LeetcodeAuthStealerService_pb2_grpc.LeetcodeAuthStealerServiceServicer
 ):
-    def __init__(self, c: Config, provider: CamouProvider) -> None:
+    def __init__(
+        self,
+        c: Config,
+        provider: CamouProvider,
+        on_failure: Callable[[], None],
+    ) -> None:
         self.provider = provider
         self.timeout_seconds = c.auth_timeout_seconds
+        self.on_failure = on_failure
+
+    def _fail(self, reason: AuthFailureReason) -> None:
+        record_auth_failure(reason)
+
+        logger.warning(
+            "Authentication failed, marking service as unalive", reason=reason
+        )
+        self.on_failure()
 
     async def Authenticate(
         self,
@@ -45,36 +60,36 @@ async def Authenticate(
                     Credentials(request.login, request.password)
                 )
         except BadCredentials:
-            record_auth_failure("bad_credentials")
+            self._fail("bad_credentials")
             await context.abort(
                 grpc.StatusCode.UNAUTHENTICATED, "Invalid LeetCode credentials"
             )
         except ManualVerificationRequired:
-            record_auth_failure("manual_verification_required")
+            self._fail("manual_verification_required")
             await context.abort(
                 grpc.StatusCode.FAILED_PRECONDITION,
                 "Manual two-step verification is required",
             )
         except AuthenticationUnavailable:
-            record_auth_failure("authentication_unavailable")
+            self._fail("authentication_unavailable")
             await context.abort(
                 grpc.StatusCode.UNAVAILABLE, "LeetCode authentication is unavailable"
             )
         except AuthenticationTimeout, TimeoutError:
-            record_auth_failure("timeout")
+            self._fail("timeout")
             await context.abort(
                 grpc.StatusCode.DEADLINE_EXCEEDED, "LeetCode authentication timed out"
             )
         except asyncio.CancelledError:
-            record_auth_failure("cancelled")
+            self._fail("cancelled")
             raise
         except Exception:
-            record_auth_failure("internal_error")
+            self._fail("internal_error")
             logger.exception("Authenticate failed")
             await context.abort(grpc.StatusCode.INTERNAL, "Authenticate failed")
 
         if not cookies.leetcode_session or not cookies.csrf_token:
-            record_auth_failure("incomplete_authentication")
+            self._fail("incomplete_authentication")
             await context.abort(
                 grpc.StatusCode.INTERNAL, "Provider returned incomplete authentication"
             )

internal/leetcode-auth-stealer-service/src/httpserve/health.py

@@ -8,8 +8,13 @@
     from httpserve.router import AppState
 
 
-async def _livez(_: web.Request) -> web.Response:
-    return web.Response(text="OK", status=200)
+async def _livez(request: web.Request) -> web.Response:
+    state: AppState = request.app["app_state"]
+
+    return web.Response(
+        text="OK" if state.live else "mr stark i dont feel so good",
+        status=200 if state.live else 503,
+    )
 
 
 async def _readyz(request: web.Request) -> web.Response:

internal/leetcode-auth-stealer-service/src/httpserve/router.py

@@ -12,6 +12,14 @@
 class AppState:
     cfg: Config
     ready: bool
+    # controlls /livez
+    # app may set this to False to trigger an unrecoverable state
+    # look at httpserve.health for how liveness works in k8s
+    live: bool = True
+
+    # use to mark unalive without having to import/pass AppState
+    def mark_unalive(self) -> None:
+        self.live = False
 
 
 def register_routes(app: web.Application, state: AppState) -> web.Application:

internal/leetcode-auth-stealer-service/src/server.py

@@ -33,12 +33,13 @@ async def serve(c: Config) -> None:
     server = grpc.aio.server(
         interceptors=[RequestIdInterceptor()],
         # more than 1 will crash the pod due to multiple browsers
-        maximum_concurrent_rpcs=1
+        maximum_concurrent_rpcs=1,
     )
     invis_provider = CamouProvider(c)
+    app_state = AppState(c, ready=False)
 
     LeetcodeAuthStealerService_pb2_grpc.add_LeetcodeAuthStealerServiceServicer_to_server(
-        AuthService(c, invis_provider), server
+        AuthService(c, invis_provider, app_state.mark_unalive), server
     )
 
     if c.grpc_reflection:
@@ -53,7 +54,6 @@ async def serve(c: Config) -> None:
     if server.add_insecure_port(address) == 0:
         raise RuntimeError(f"Unable to bind gRPC server to {address}")
 
-    app_state = AppState(c, ready=False)
     app = register_routes(web.Application(), app_state)
     runner = web.AppRunner(app)
 

src/main/java/org/patinanetwork/codebloom/scheduled/auth/LeetcodeAuthStealer.java

@@ -62,7 +62,7 @@ public LeetcodeAuthStealer(
      * authentication key from Leetcode. That code is stored in the database and can then be used to run authenticated
      * queries such as being used to retrieve code from our user submissions.
      */
-    @Scheduled(initialDelay = 0, fixedDelay = 1, timeUnit = TimeUnit.HOURS)
+    @Scheduled(initialDelay = 0, fixedDelay = 15, timeUnit = TimeUnit.MINUTES)
     public void stealAuthCookie() {
         boolean acquired = LOCK.writeLock().tryLock();
         if (!acquired) {