fix(webapp,run-store): make run-ops sharding failures visible and unroutable ids a 404

apps/webapp/app/routes/@.runs.$runParam.ts

@@ -8,6 +8,7 @@ import { redirectWithErrorMessage } from "~/models/message.server";
 import { requireUser } from "~/services/session.server";
 import { impersonate, rootPath, v3RunPath, v3RunSpanPath } from "~/utils/pathBuilder";
 import { findBufferedRunRedirectInfo } from "~/v3/mollifier/syntheticRedirectInfo.server";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 const ParamsSchema = z.object({
   runParam: z.string(),
@@ -33,17 +34,21 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     );
   }
 
-  const run = await runStore.findRun(
-    {
-      friendlyId: runParam,
-    },
-    {
-      select: {
-        spanId: true,
-        runtimeEnvironmentId: true,
-      },
-    },
-    prisma
+  const run = await undefinedOnUnroutableId(
+    () =>
+      runStore.findRun(
+        {
+          friendlyId: runParam,
+        },
+        {
+          select: {
+            spanId: true,
+            runtimeEnvironmentId: true,
+          },
+        },
+        prisma
+      ),
+    { runParam: params.runParam ?? params.runId }
   );
 
   if (!run) {

apps/webapp/app/routes/api.v1.batches.$batchParam.results.ts

@@ -5,6 +5,7 @@ import { runOpsLegacyReplica, runOpsNewReplica, runOpsSplitReadEnabled } from "~
 import { ApiBatchResultsPresenter } from "~/presenters/v3/ApiBatchResultsPresenter.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server";
 
 const ParamsSchema = z.object({
   /* This is the batch friendly ID */
@@ -42,6 +43,14 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
 
       return json(result);
     } catch (error) {
+      const unroutable = unroutableIdResponse(error);
+      if (unroutable) {
+        logger.warn("Unroutable batch id on batch results", {
+          error: error instanceof Error ? error.message : error,
+        });
+        return unroutable;
+      }
+
       logger.error("Failed to load batch results", { error });
       return json({ error: "Something went wrong, please try again." }, { status: 500 });
     }

apps/webapp/app/routes/api.v1.runs.$runId.tags.ts

@@ -10,6 +10,7 @@ import { logger } from "~/services/logger.server";
 import { publishChangeRecord } from "~/services/realtime/runChangeNotifierInstance.server";
 import { mutateWithFallback } from "~/v3/mollifier/mutateWithFallback.server";
 import { runStore } from "~/v3/runStore.server";
+import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server";
 
 // Pull the existing tags out of a buffer entry's serialised payload so
 // the buffer-path response can dedup against them, matching the
@@ -137,6 +138,14 @@ export async function action({ request, params }: ActionFunctionArgs) {
     }
     return outcome.response;
   } catch (error) {
+    const unroutable = unroutableIdResponse(error);
+    if (unroutable) {
+      logger.warn("Unroutable run id on run tags", {
+        error: error instanceof Error ? error.message : error,
+      });
+      return unroutable;
+    }
+
     logger.error("Failed to add run tags", { error });
     return json({ error: "Something went wrong, please try again." }, { status: 500 });
   }

apps/webapp/app/routes/api.v1.runs.$runParam.reschedule.ts

@@ -12,6 +12,7 @@ import { RescheduleTaskRunService } from "~/v3/services/rescheduleTaskRun.server
 import { mutateWithFallback } from "~/v3/mollifier/mutateWithFallback.server";
 import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
 import { parseDelay } from "~/utils/delays";
+import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server";
 
 const ParamsSchema = z.object({
   runParam: z.string(),
@@ -156,6 +157,14 @@ export async function action({ request, params }: ActionFunctionArgs) {
     if (error instanceof ServiceValidationError) {
       return json({ error: error.message }, { status: 400 });
     }
+    const unroutable = unroutableIdResponse(error);
+    if (unroutable) {
+      logger.warn("Unroutable run id on reschedule", {
+        error: error instanceof Error ? error.message : error,
+      });
+      return unroutable;
+    }
+
     logger.error("Failed to reschedule run", { error });
     return json({ error: "Something went wrong, please try again." }, { status: 500 });
   }

apps/webapp/app/routes/api.v1.runs.$runParam.result.ts

@@ -5,6 +5,7 @@ import { ApiRunResultPresenter } from "~/presenters/v3/ApiRunResultPresenter.ser
 import { runOpsLegacyReplica, runOpsNewReplica, runOpsSplitReadEnabled } from "~/db.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
+import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server";
 
 const ParamsSchema = z.object({
   /* This is the run friendly ID */
@@ -41,6 +42,14 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
 
     return json(result);
   } catch (error) {
+    const unroutable = unroutableIdResponse(error);
+    if (unroutable) {
+      logger.warn("Unroutable run id on run result", {
+        error: error instanceof Error ? error.message : error,
+      });
+      return unroutable;
+    }
+
     logger.error("Failed to load run result", { error });
     return json({ error: "Something went wrong, please try again." }, { status: 500 });
   }

apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts

@@ -6,6 +6,7 @@ import { env } from "~/env.server";
 import { processWaitpointCompletionPacket } from "~/runEngine/concerns/waitpointCompletionPacket.server";
 import { verifyHttpCallbackHash } from "~/services/httpCallback.server";
 import { logger } from "~/services/logger.server";
+import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
 import { engine } from "~/v3/runEngine.server";
 import { runStore } from "~/v3/runStore.server";
@@ -102,6 +103,19 @@ export async function action({ request, params }: ActionFunctionArgs) {
       { status: 200 }
     );
   } catch (error) {
+    // Same as the complete route: the waitpoint id comes off the URL, so an unconfigured shard
+    // key is caller-supplied input and must answer 404 rather than 500. This route is a bare
+    // Remix action, so the api-builder boundary never sees the error — answer it here.
+    const unroutable = unroutableIdResponse(error);
+    if (unroutable) {
+      // Same reason as the complete route: a silent 404 would hide a dropped shard key.
+      logger.warn("Unroutable waitpoint id on HTTP callback", {
+        waitpointFriendlyId: params.waitpointFriendlyId,
+        error: error instanceof Error ? error.message : error,
+      });
+      return unroutable;
+    }
+
     logger.error("Failed to complete HTTP callback", { error });
     throw json({ error: "Failed to complete HTTP callback" }, { status: 500 });
   }

apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts

@@ -10,6 +10,7 @@ import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 import { processWaitpointCompletionPacket } from "~/runEngine/concerns/waitpointCompletionPacket.server";
 import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server";
+import { unroutableIdResponse } from "~/services/routeBuilders/unroutableId.server";
 import { engine } from "~/v3/runEngine.server";
 import { runStore } from "~/v3/runStore.server";
 
@@ -87,6 +88,19 @@ const { action, loader } = createActionApiRoute(
       // client gets the correct status code instead of a 500, and we don't log them as errors.
       if (error instanceof Response) throw error;
 
+      // A caller-supplied id naming a shard this topology has no store for cannot be routed,
+      // so it is a 404 like an absent token — not the 500 this catch would otherwise answer.
+      const unroutable = unroutableIdResponse(error);
+      if (unroutable) {
+        // Logged so a shard key dropped from an append-only config still alarms, rather than
+        // every live token on it quietly answering "not found".
+        logger.warn("Unroutable waitpoint id on token completion", {
+          waitpointFriendlyId: params.waitpointFriendlyId,
+          error: error instanceof Error ? error.message : error,
+        });
+        throw unroutable;
+      }
+
       logger.error("Failed to complete waitpoint token", {
         error:
           error instanceof Error

apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts

@@ -6,6 +6,7 @@ import { requireUserId } from "~/services/session.server";
 import { ProjectParamSchema, v3RunPath } from "~/utils/pathBuilder";
 import { runStore } from "~/v3/runStore.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 const ParamSchema = ProjectParamSchema.extend({
   runParam: z.string(),
@@ -15,16 +16,20 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const userId = await requireUserId(request);
   const { organizationSlug, projectParam, runParam } = ParamSchema.parse(params);
 
-  const run = await runStore.findRun(
-    {
-      friendlyId: runParam,
-    },
-    {
-      select: {
-        projectId: true,
-        runtimeEnvironmentId: true,
-      },
-    }
+  const run = await undefinedOnUnroutableId(
+    () =>
+      runStore.findRun(
+        {
+          friendlyId: runParam,
+        },
+        {
+          select: {
+            projectId: true,
+            runtimeEnvironmentId: true,
+          },
+        }
+      ),
+    { runParam: params.runParam ?? params.runId }
   );
 
   if (!run) {

apps/webapp/app/routes/projects.v3.$projectRef.runs.$runParam.ts

@@ -5,6 +5,7 @@ import { requireUserId } from "~/services/session.server";
 import { v3RunSpanPath } from "~/utils/pathBuilder";
 import { runStore } from "~/v3/runStore.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 const ParamsSchema = z.object({
   projectRef: z.string(),
@@ -36,18 +37,22 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
     return new Response("Not found", { status: 404 });
   }
 
-  const run = await runStore.findRun(
-    {
-      friendlyId: validatedParams.runParam,
-    },
-    {
-      select: {
-        friendlyId: true,
-        spanId: true,
-        runtimeEnvironmentId: true,
-      },
-    },
-    prisma
+  const run = await undefinedOnUnroutableId(
+    () =>
+      runStore.findRun(
+        {
+          friendlyId: validatedParams.runParam,
+        },
+        {
+          select: {
+            friendlyId: true,
+            spanId: true,
+            runtimeEnvironmentId: true,
+          },
+        },
+        prisma
+      ),
+    { runParam: params.runParam ?? params.runId }
   );
 
   if (!run) {

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.sessions.$sessionId.$io.ts

@@ -13,6 +13,7 @@ import {
 import { getRealtimeStreamInstance } from "~/services/realtime/v1StreamsGlobal.server";
 import { requireUserId } from "~/services/session.server";
 import { EnvironmentParamSchema } from "~/utils/pathBuilder";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 const ParamsSchema = z.object({
   runParam: z.string(),
@@ -61,8 +62,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
   // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab subscription
   // (useRealtimeStream surfaces the error and does not auto-retry). Re-read the primary on a miss.
   const run =
-    (await runStore.findRun(runWhere, runArgs, $replica)) ??
-    (await runStore.findRunOnPrimary(runWhere, runArgs));
+    (await undefinedOnUnroutableId(() => runStore.findRun(runWhere, runArgs, $replica), {
+      runParam: params.runParam,
+    })) ??
+    (await undefinedOnUnroutableId(() => runStore.findRunOnPrimary(runWhere, runArgs), {
+      runParam: params.runParam,
+    }));
 
   if (!run) {
     return new Response("Run not found", { status: 404 });

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.$streamId.ts

@@ -8,6 +8,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { requireUserId } from "~/services/session.server";
 import { EnvironmentParamSchema } from "~/utils/pathBuilder";
 import { runStore } from "~/v3/runStore.server";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 const ParamsSchema = z.object({
   runParam: z.string(),
@@ -60,8 +61,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
   // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab subscription
   // (useRealtimeStream surfaces the error and does not auto-retry). Re-read the primary on a miss.
   const run =
-    (await runStore.findRun(runWhere, runArgs, $replica)) ??
-    (await runStore.findRunOnPrimary(runWhere, runArgs));
+    (await undefinedOnUnroutableId(() => runStore.findRun(runWhere, runArgs, $replica), {
+      runParam: params.runParam,
+    })) ??
+    (await undefinedOnUnroutableId(() => runStore.findRunOnPrimary(runWhere, runArgs), {
+      runParam: params.runParam,
+    }));
 
   if (!run) {
     return new Response("Run not found", { status: 404 });

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.realtime.v1.streams.$runId.input.$streamId.ts

@@ -8,6 +8,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { requireUserId } from "~/services/session.server";
 import { EnvironmentParamSchema } from "~/utils/pathBuilder";
 import { runStore } from "~/v3/runStore.server";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 const ParamsSchema = z.object({
   runParam: z.string(),
@@ -62,8 +63,12 @@ export async function loader({ request, params }: LoaderFunctionArgs) {
   // Replica lag can null out a live run; a spurious 404 breaks the dashboard Agent tab input-stream
   // subscription (useRealtimeStream surfaces the error, no auto-retry). Re-read the primary on a miss.
   const run =
-    (await runStore.findRun(runWhere, runArgs, $replica)) ??
-    (await runStore.findRunOnPrimary(runWhere, runArgs));
+    (await undefinedOnUnroutableId(() => runStore.findRun(runWhere, runArgs, $replica), {
+      runParam: params.runParam,
+    })) ??
+    (await undefinedOnUnroutableId(() => runStore.findRunOnPrimary(runWhere, runArgs), {
+      runParam: params.runParam,
+    }));
 
   if (!run) {
     return new Response("Run not found", { status: 404 });

apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts

@@ -15,6 +15,7 @@ import {
   type TraceExportContext,
 } from "~/v3/eventRepository/traceExport.server";
 import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
+import { undefinedOnUnroutableId } from "~/v3/runOpsMigration/unroutableRead.server";
 
 export async function loader({ params, request }: LoaderFunctionArgs) {
   const user = await requireUser(request);
@@ -31,20 +32,24 @@ export async function loader({ params, request }: LoaderFunctionArgs) {
   // Run-ops read keyed by friendlyId only (routes to the owning DB by residency). Org
   // membership is a control-plane concern resolved separately below — joining it here is a
   // cross-DB join that returns nothing once the run lives in run-ops.
-  let run = await runStore.findRun(
-    { friendlyId: parsedParams.runParam },
-    {
-      select: {
-        friendlyId: true,
-        traceId: true,
-        organizationId: true,
-        runtimeEnvironmentId: true,
-        createdAt: true,
-        completedAt: true,
-        taskEventStore: true,
-        taskIdentifier: true,
-      },
-    }
+  let run = await undefinedOnUnroutableId(
+    () =>
+      runStore.findRun(
+        { friendlyId: parsedParams.runParam },
+        {
+          select: {
+            friendlyId: true,
+            traceId: true,
+            organizationId: true,
+            runtimeEnvironmentId: true,
+            createdAt: true,
+            completedAt: true,
+            taskEventStore: true,
+            taskIdentifier: true,
+          },
+        }
+      ),
+    { runParam: parsedParams.runParam }
   );
 
   // Authorize on the control-plane DB: the user must be a member of the run's org. A