diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 074bc39d760..501f478fdf6 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -17,6 +17,8 @@ import { LocaleContextProvider } from "./components/primitives/LocaleProvider"; import type { OperatingSystemPlatform } from "./components/primitives/OperatingSystemProvider"; import { OperatingSystemContextProvider } from "./components/primitives/OperatingSystemProvider"; import { assertRunOpsSplitSentinel, Prisma } from "./db.server"; +import { assertSnapshotStoreBootFromEnv } from "./v3/snapshotStoreBoot.server"; +import { registerSnapshotStoreWiring } from "./v3/snapshotStoreWiring.server"; import { env } from "./env.server"; import { eventLoopMonitor, eventLoopUtilizationMonitor } from "./eventLoopMonitor.server"; import { logger } from "./services/logger.server"; @@ -325,6 +327,18 @@ singleton("AssertRunOpsSplitSentinel", () => { return true; }); +singleton("SnapshotStoreWiring", registerSnapshotStoreWiring); + +// Ordered after the wiring above: the boot check asserts the repair binding is set, and the +// binding is what the wiring installs. +singleton("AssertSnapshotStoreBoot", () => { + assertSnapshotStoreBootFromEnv().catch((error) => { + logger.error("Snapshot store boot check failed; refusing to start", { error }); + process.exit(1); + }); + return true; +}); + singleton("RunEngineEventBusHandlers", registerRunEngineEventBusHandlers); singleton("SetupBatchQueueCallbacks", setupBatchQueueCallbacks); // Attach the realtime run-changed publish delegations to the engine event bus. diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 2b1fba86980..725df812a1d 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1307,6 +1307,72 @@ const EnvironmentSchema = z .string() .default(process.env.REDIS_TLS_DISABLED ?? "false"), + // Execution-snapshot store. MODE here is only the FLOOR: the operational dial is the + // snapshotStoreMode feature flag, so it can move without a deploy. + RUN_ENGINE_SNAPSHOT_STORE_MODE: z + .enum(["off", "dual-write", "redis-read", "redis-only"]) + .default("off"), + RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS: z.coerce + .number() + .int() + .default(72 * 60 * 60 * 1000), + RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS: z.coerce + .number() + .int() + .default(24 * 60 * 60 * 1000), + RUN_ENGINE_SNAPSHOT_STORE_CONFIRM_ORPHAN_AFTER_MS: z.coerce + .number() + .int() + .default(2 * 60 * 60 * 1000), + RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE: z.string().default("0 */6 * * *"), + RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS: z.coerce.number().int().min(0).default(60_000), + // An existing run costs ~4 serial round trips and the orphan-marker clear cannot be batched + // (cross-slot pipelines are rejected), so a full pass is hours, not minutes. A budget that + // truncates every pass stops rule 2 converging, because it needs consecutive sightings. + RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS: z.coerce + .number() + .int() + .positive() + .default(10_800_000), + /** + * RETIRED. The hard stop is the snapshotStoreHalt feature flag and nothing else: an environment + * variable converged over a rolling deploy rather than a flag interval, and during that window a + * stopped process skips a transition while a running one asserts a head that was never written. + * + * Kept in the schema for one purpose only, so boot can REFUSE to start when it is still set to + * "1". A variable that no longer stops anything leaves an operator believing the mirror is + * stopped while it runs. Nothing else reads it, and a value of "0" carries no intent so it is + * ignored. + */ + RUN_ENGINE_SNAPSHOT_STORE_HALT: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS: z.coerce + .number() + .int() + .positive() + .default(30_000), + RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX: z.coerce + .number() + .int() + .positive() + .default(10_000), + RUN_ENGINE_SNAPSHOT_STORE_RUN_ORG_CACHE_MAX: z.coerce.number().int().positive().default(50_000), + // No fallback to REDIS_*: this is a distinct durable endpoint and must be set explicitly, or + // execution state silently lands on the general-purpose cache. + RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT: z.coerce.number().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_USERNAME: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_PASSWORD: z.string().optional(), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED: z.string().default("false"), + RUN_ENGINE_SNAPSHOT_STORE_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + // Fails an append fast rather than letting it wait on an unreachable endpoint. Postgres is + // authoritative below the final dial position, so a refused append costs a mirrored write; a + // blocked one costs the request. + RUN_ENGINE_SNAPSHOT_STORE_REDIS_COMMAND_TIMEOUT_MS: z.coerce + .number() + .int() + .positive() + .default(500), + RUN_ENGINE_DEV_PRESENCE_REDIS_HOST: z .string() .optional() diff --git a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts index e9da02effd9..ce72b0077d4 100644 --- a/apps/webapp/app/routes/admin.api.v1.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.feature-flags.ts @@ -2,14 +2,20 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { applyGlobalGracedFlips, - makeSetMultipleFlags, + setGlobalFeatureFlagsTransactional, + stampGlobalModeLatchForMerge, touchesGracedGroup, withoutDerivedKeys, } from "~/v3/featureFlags.server"; -import { validatePartialFeatureFlags } from "~/v3/featureFlags"; +import { validatePartialFeatureFlags, FEATURE_FLAG } from "~/v3/featureFlags"; +import { + globalOnlySnapshotStoreFlagError, + snapshotStoreFlagSaveError, +} from "~/v3/snapshotStoreFlagGuard.server"; export async function action({ request }: ActionFunctionArgs) { await requireAdminApiRequest(request); @@ -30,6 +36,22 @@ export async function action({ request }: ActionFunctionArgs) { ); } + const globalOnlyError = globalOnlySnapshotStoreFlagError(body as Record); + if (globalOnlyError) { + return json({ error: globalOnlyError }, { status: 400 }); + } + + const snapshotStoreError = snapshotStoreFlagSaveError(body as Record, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + // Read from the live registry, not from the payload: the latch must ALREADY be true before + // anything can be enabled, or a run born in the gap would be resident with its transitions + // skipped. + everEnabled: globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreEverEnabled] === true, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + // Both the strip and the branch derive from the graced-group table, so adding a group needs // no edit here. Naming the keys inline is how a new group ends up writing its stamp straight // from the request body, with no lock. @@ -37,9 +59,12 @@ export async function action({ request }: ActionFunctionArgs) { typeof validationResult.data >; - const updatedFlags = touchesGracedGroup(requestedFlags) - ? await applyGlobalGracedFlips(prisma, requestedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS) - : await makeSetMultipleFlags(prisma)(requestedFlags); + // Stamp the one-way global mode latch before the write, so neither merge branch bypasses it. + const stampedFlags = await stampGlobalModeLatchForMerge(prisma, requestedFlags); + + const updatedFlags = touchesGracedGroup(stampedFlags) + ? await applyGlobalGracedFlips(prisma, stampedFlags, env.RUN_OPS_MINT_FLIP_GRACE_MS) + : await setGlobalFeatureFlagsTransactional(prisma, stampedFlags); return json({ success: true, diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts index db1524f8551..d58f67bfc6d 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.feature-flags.ts @@ -6,8 +6,17 @@ import { env } from "~/env.server"; import { prisma } from "~/db.server"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; +import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server"; +import { snapshotStoreOrgCensus } from "~/v3/snapshotStoreOrgCensus.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; -import { validatePartialFeatureFlags } from "~/v3/featureFlags"; +import { + FEATURE_FLAG, + stampSnapshotStoreOrgEverEnabled, + validatePartialFeatureFlags, + withoutOrgForbiddenSnapshotKeys, +} from "~/v3/featureFlags"; import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; const ParamsSchema = z.object({ @@ -71,9 +80,22 @@ export async function action({ request, params }: ActionFunctionArgs) { const { runOpsMintKindPrev: _ignoredPrev, runOpsMintKindFlippedAt: _ignoredFlippedAt, - ...requestedFlags + ...rawRequestedFlags } = validationResult.data; + const requestedFlags = withoutOrgForbiddenSnapshotKeys(rawRequestedFlags); + + const snapshotStoreError = snapshotStoreFlagSaveError(requestedFlags, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + // Read from the live registry, not from the payload: the latch must ALREADY be true before + // anything can be enabled, or a run born in the gap would be resident with its transitions + // skipped. + everEnabled: globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreEverEnabled] === true, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override // is graced from the currently-effective global kind, not the hardcoded default "cuid". const globalFlags = (await getGlobalFlags()) as Record; @@ -108,6 +130,12 @@ export async function action({ request, params }: ActionFunctionArgs) { env.RUN_OPS_MINT_FLIP_GRACE_MS ); + // One-way per-org residency latch, exactly as the v2 route does. Without it a run born after + // this enable is resident but the census keeps classifying the org definitely-never-enabled, so + // its transitions are skipped and its Redis head freezes. ORed against the locked existing value + // so a save back to off never clears it. + stampSnapshotStoreOrgEverEnabled(existingRaw, mergedFlags); + return tx.organization.update({ where: { id: organizationId, @@ -129,6 +157,10 @@ export async function action({ request, params }: ActionFunctionArgs) { // Org feature flags are embedded in every env of the org; drop all its cached env rows. controlPlaneResolver.invalidateOrganization(organizationId); + invalidateSnapshotStoreOrgMode(organizationId); + // Refresh the census in THIS process at once, as the v2 route does, so a just-enabled org stops + // reading as definitely-never-enabled here immediately. Other pods lag at most the reload interval. + void snapshotStoreOrgCensus.refresh(); const updatedFlagsResult = updatedOrganization.featureFlags ? validatePartialFeatureFlags(updatedOrganization.featureFlags as Record) diff --git a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts index 0071054bd3e..01b30dae43a 100644 --- a/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts +++ b/apps/webapp/app/routes/admin.api.v2.orgs.$organizationId.feature-flags.ts @@ -6,11 +6,18 @@ import { env } from "~/env.server"; import { prisma } from "~/db.server"; import { requireUser } from "~/services/session.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; +import { snapshotStoreFlagSaveError } from "~/v3/snapshotStoreFlagGuard.server"; +import { invalidateSnapshotStoreOrgMode } from "~/v3/snapshotStoreMode.server"; +import { snapshotStoreOrgCensus } from "~/v3/snapshotStoreOrgCensus.server"; import { selectMintBaselineSource, stampMintKindFlip } from "~/v3/runOpsMigration/mintFlipGrace"; import { flags as getGlobalFlags } from "~/v3/featureFlags.server"; import { + clearedOrgFlagsPreservingLatch, FEATURE_FLAG, + stampSnapshotStoreOrgEverEnabled, validatePartialFeatureFlags, + withoutOrgForbiddenSnapshotKeys, getAllFlagControlTypes, } from "~/v3/featureFlags"; import { featuresForRequest } from "~/features.server"; @@ -109,20 +116,37 @@ export async function action({ request, params }: ActionFunctionArgs) { body === null || (typeof body === "object" && !Array.isArray(body) && Object.keys(body).length === 0) ) { - // Clear all flags. No grace stamp (nothing to flip) and no read-then-write race. - try { - await prisma.organization.update({ + // Clear all flags, but preserve the one-way per-org residency latch so an ever-enabled org can + // never drop out of the census. Locked read-then-write so a concurrent enabling save (which also + // takes FOR UPDATE) can't slip a latch in between the read and the wipe. + const updated = await prisma.$transaction(async (tx) => { + const rows = await tx.$queryRaw<{ featureFlags: unknown }[]>` + SELECT "featureFlags" FROM "Organization" WHERE "id" = ${organizationId} FOR UPDATE`; + + if (rows.length === 0) { + return false; + } + + const preserved = clearedOrgFlagsPreservingLatch( + rows[0].featureFlags as Record | null + ); + + await tx.organization.update({ where: { id: organizationId }, - data: { featureFlags: Prisma.JsonNull }, + data: { + featureFlags: preserved === null ? Prisma.JsonNull : (preserved as Prisma.InputJsonValue), + }, }); - } catch (e) { - if (e instanceof Prisma.PrismaClientKnownRequestError && e.code === "P2025") { - throw new Response("Organization not found", { status: 404 }); - } - throw e; + + return true; + }); + + if (!updated) { + throw new Response("Organization not found", { status: 404 }); } controlPlaneResolver.invalidateOrganization(organizationId); + invalidateSnapshotStoreOrgMode(organizationId); return json({ success: true }); } @@ -138,9 +162,21 @@ export async function action({ request, params }: ActionFunctionArgs) { const { runOpsMintKindPrev: _ignoredPrev, runOpsMintKindFlippedAt: _ignoredFlippedAt, - ...requestedFlags + ...rawRequestedFlags } = validationResult.data; + const requestedFlags = withoutOrgForbiddenSnapshotKeys(rawRequestedFlags); + + const snapshotStoreError = snapshotStoreFlagSaveError(requestedFlags, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + // Read from the live registry, not the payload: the latch must ALREADY be true before anything + // can be enabled, or a run born in the gap is resident with its transitions skipped. + everEnabled: globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreEverEnabled] === true, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + // Seed the flip baseline from the current GLOBAL mint flags so an org's FIRST per-org override // is graced from the currently-effective global kind, not the hardcoded default "cuid". const globalFlags = (await getGlobalFlags()) as Record; @@ -167,6 +203,9 @@ export async function action({ request, params }: ActionFunctionArgs) { env.RUN_OPS_MINT_FLIP_GRACE_MS ); + // One-way per-org residency latch, ORed against the locked existing value so it never clears. + stampSnapshotStoreOrgEverEnabled(existingRaw, stamped); + await tx.organization.update({ where: { id: organizationId }, data: { featureFlags: stamped as Prisma.InputJsonValue }, @@ -181,6 +220,11 @@ export async function action({ request, params }: ActionFunctionArgs) { // Org feature flags are embedded in every env of the org; drop all its cached env rows. controlPlaneResolver.invalidateOrganization(organizationId); + invalidateSnapshotStoreOrgMode(organizationId); + // Refresh the census in THIS process at once, so a just-enabled org stops reading as + // definitely-never-enabled here immediately. Other pods lag at most the reload interval. This only + // shrinks the enabling-edge transient at the transition skip; it does not make it airtight. + void snapshotStoreOrgCensus.refresh(); return json({ success: true }); } diff --git a/apps/webapp/app/routes/admin.feature-flags.tsx b/apps/webapp/app/routes/admin.feature-flags.tsx index e987812f520..d3319f8a064 100644 --- a/apps/webapp/app/routes/admin.feature-flags.tsx +++ b/apps/webapp/app/routes/admin.feature-flags.tsx @@ -2,6 +2,7 @@ import { useFetcher } from "@remix-run/react"; import { useEffect, useState } from "react"; import stableStringify from "json-stable-stringify"; import { json } from "@remix-run/server-runtime"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { LockClosedIcon } from "@heroicons/react/20/solid"; @@ -17,6 +18,10 @@ import { lockedFlagsInPayload, validatePartialFeatureFlags, } from "~/v3/featureFlags"; +import { + globalOnlySnapshotStoreFlagError, + snapshotStoreFlagSaveError, +} from "~/v3/snapshotStoreFlagGuard.server"; import { flags as getGlobalFlags, replaceGlobalFeatureFlags } from "~/v3/featureFlags.server"; import { featuresForRequest } from "~/features.server"; import { Button } from "~/components/primitives/Buttons"; @@ -129,6 +134,22 @@ export const action = dashboardAction( ); } + const globalOnlyError = globalOnlySnapshotStoreFlagError(parsed.data.flags); + if (globalOnlyError) { + return json({ error: globalOnlyError }, { status: 400 }); + } + + const snapshotStoreError = snapshotStoreFlagSaveError(parsed.data.flags, { + redisHostConfigured: !!env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + // Read from the live registry, not from the payload: the latch must ALREADY be true before + // anything can be enabled, or a run born in the gap would be resident with its transitions + // skipped. + everEnabled: globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreEverEnabled] === true, + }); + if (snapshotStoreError) { + return json({ error: snapshotStoreError }, { status: 400 }); + } + await replaceGlobalFeatureFlags(prisma, { requestedFlags: validationResult.data as Record, catalogKeys: Object.keys(getAllFlagControlTypes()) as FeatureFlagKey[], diff --git a/apps/webapp/app/v3/cohortMetricLabel.server.ts b/apps/webapp/app/v3/cohortMetricLabel.server.ts new file mode 100644 index 00000000000..b48a99f4348 --- /dev/null +++ b/apps/webapp/app/v3/cohortMetricLabel.server.ts @@ -0,0 +1,8 @@ +// Bounds per-org metric cardinality: only cohort members get a distinct label, everything else +// (undefined included) collapses to "other" — at most cohort size plus one series. +export function cohortMetricLabel( + organizationId: string | undefined, + isCohortMember: (organizationId: string) => boolean +): string { + return organizationId !== undefined && isCohortMember(organizationId) ? organizationId : "other"; +} diff --git a/apps/webapp/app/v3/featureFlags.server.ts b/apps/webapp/app/v3/featureFlags.server.ts index 152a14b6496..b3ed868141e 100644 --- a/apps/webapp/app/v3/featureFlags.server.ts +++ b/apps/webapp/app/v3/featureFlags.server.ts @@ -7,6 +7,7 @@ import { FeatureFlagCatalog, GLOBAL_LOCKED_FLAGS, GRACED_FLAG_GROUPS, + stampSnapshotStoreGlobalModeEverEnabled, validatePartialFeatureFlags, } from "~/v3/featureFlags"; import { env } from "~/env.server"; @@ -295,6 +296,52 @@ export async function applyGlobalGracedFlips( return applied; } +// Plain merge write in one transaction, so an ordered mode+latch pair commits atomically instead of +// as two upserts a crash could split. The non-graced sibling of applyGlobalGracedFlips. +export async function setGlobalFeatureFlagsTransactional( + client: PrismaClient, + requestedFlags: Partial> +): Promise<{ key: string; value: any }[]> { + const applied = await $transaction(client, "setGlobalFeatureFlags", (tx) => + makeSetMultipleFlags(tx)(requestedFlags) + ); + + if (!applied) { + throw new Error("setGlobalFeatureFlagsTransactional: transaction did not complete"); + } + return applied; +} + +// One-way global mode latch for the merge-semantics JSON admin API, whose two write branches would +// otherwise bypass the stamp. Reading outside a transaction is safe: a one-way latch only ever +// races to write true twice. +export async function stampGlobalModeLatchForMerge>( + client: PrismaClientOrTransaction, + requestedFlags: T +): Promise { + const stored = await client.featureFlag.findFirst({ + where: { key: FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled }, + select: { value: true }, + }); + // The stamp is the only writer, so an operator-supplied value (a `false` above all) never counts. + const latchKey = FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled; + const stamped = { ...requestedFlags }; + delete stamped[latchKey]; + const result = stampSnapshotStoreGlobalModeEverEnabled({ [latchKey]: stored?.value }, stamped); + + // Order the latch BEFORE the mode. makeSetMultipleFlags upserts in insertion order with no + // enclosing transaction, so a crash between the two must leave latch=true with the mode possibly + // still off (the safe direction that makes 10c not skip), never mode=non-off with the latch absent. + if (result[latchKey] === true) { + const reordered: Record = { [latchKey]: result[latchKey] }; + for (const [key, value] of Object.entries(result)) { + if (key !== latchKey) reordered[key] = value; + } + return reordered as T; + } + return result as T; +} + // Replace-semantics write for the global admin flags page: submitted flags upsert, omitted ones // delete unless protected. One transaction covers the stamp, the upserts and the deletes, so a // save cannot half-apply. @@ -315,6 +362,8 @@ export async function replaceGlobalFeatureFlags( } ): Promise { const requestedFlags = withoutDerivedKeys(params.requestedFlags); + // System-set latch: the save path is its only writer, so an operator-supplied value never counts. + delete requestedFlags[FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled]; // A locked flag absent from the payload means the page never offered it, not that the admin // unset it, so it survives. Only a self-hosted page that says it unlocked them may delete one. @@ -326,6 +375,15 @@ export async function replaceGlobalFeatureFlags( await lockGracedGroups(tx); const stamped = await stampGracedGroups(tx, requestedFlags, params.graceMs); + // One-way global mode latch: OR the stored value with "enabling now" so a save back to off + // never clears it. Forced into the write set and out of the sweep below. + const latchKey = FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled; + const storedLatch = await tx.featureFlag.findFirst({ + where: { key: latchKey }, + select: { value: true }, + }); + stampSnapshotStoreGlobalModeEverEnabled({ [latchKey]: storedLatch?.value }, stamped); + const toWrite: Record = {}; const keysToDelete: string[] = []; @@ -350,6 +408,13 @@ export async function replaceGlobalFeatureFlags( } } + // Never delete the latch, even on a self-hosted unlock: write it and drop it from the sweep. + if (stamped[latchKey] === true) { + toWrite[latchKey] = true; + const deleteIndex = keysToDelete.indexOf(latchKey); + if (deleteIndex !== -1) keysToDelete.splice(deleteIndex, 1); + } + // One round trip to learn the stored values, then a write only for what actually differs. // makeSetMultipleFlags upserts sequentially, so an unchanged flag costs a round trip for // nothing, and this transaction is interactive and holds a pooled connection. diff --git a/apps/webapp/app/v3/featureFlags.ts b/apps/webapp/app/v3/featureFlags.ts index 3dccd5c4cc7..f7cb519b2a9 100644 --- a/apps/webapp/app/v3/featureFlags.ts +++ b/apps/webapp/app/v3/featureFlags.ts @@ -50,6 +50,26 @@ export const FEATURE_FLAG = { // System-wide kill switch for additional (scoped) environment API-key lookup. // Defaults off; enable during rollout once the new lookup path is trusted. additionalApiKeyLookupEnabled: "additionalApiKeyLookupEnabled", + // The execution-snapshot store rollout dial. A flag rather than an environment variable because + // a sustained append failure burns a task attempt per transition, so dial-down is a correctness + // control and cannot wait for a deploy. + snapshotStoreMode: "snapshotStoreMode", + // The hard stop for the execution-snapshot store, deployment-wide. Separate from the dial because + // the dial governs births only: turning it down cannot stop a resident run from mirroring, and + // must not, or every resident head freezes while Postgres advances. + snapshotStoreHalt: "snapshotStoreHalt", + // Per-org override, read from the org blob only. Deliberately narrower than the global key: + // snapshot reads are global, so an org at a read position would read state its own writes never + // created. Stripped from org payloads by withoutOrgForbiddenSnapshotKeys. + snapshotStoreOrgMode: "snapshotStoreOrgMode", + // One-way residency latch. See the catalog entry below. + snapshotStoreEverEnabled: "snapshotStoreEverEnabled", + // Per-org one-way residency latch, the per-org sibling of snapshotStoreEverEnabled. System-set on + // the org save path when the org dial first moves past `off`, never cleared. See the catalog entry. + snapshotStoreOrgEverEnabled: "snapshotStoreOrgEverEnabled", + // Global one-way latch: true once the global dial has been non-off at least once. System-set on + // the global save paths, never cleared. See the catalog entry below. + snapshotStoreGlobalModeEverEnabled: "snapshotStoreGlobalModeEverEnabled", } as const; export const FeatureFlagCatalog = { @@ -163,6 +183,33 @@ export const FeatureFlagCatalog = { [FEATURE_FLAG.additionalApiKeysEnabled]: z.boolean(), [FEATURE_FLAG.additionalApiKeyIssuanceEnabled]: z.boolean(), [FEATURE_FLAG.additionalApiKeyLookupEnabled]: z.boolean(), + [FEATURE_FLAG.snapshotStoreMode]: z.enum(["off", "dual-write", "redis-read", "redis-only"]), + [FEATURE_FLAG.snapshotStoreOrgMode]: z.enum(["off", "dual-write", "redis-read", "redis-only"]), + /** + * Whether this deployment has EVER had the store enabled. One way: set when the first dial or + * per-organisation override moves past `off`, and never cleared automatically. + * + * It exists so that `off` means genuinely inert before a ramp. A transition has to ask whether its + * run is resident, and the keyspace is the only record of that, so at `off` after a ramp every + * transition must still ask or a resident run's head freezes. Before any ramp nothing CAN be + * resident, so the question has one possible answer and asking it is pure cost: measured at 2 per + * cent with a healthy endpoint and four times the run duration with a slow one. + * + * Strict boolean, like the other kill switches: a stringified "false" read as true would put the + * whole fleet back on the run path. + */ + [FEATURE_FLAG.snapshotStoreEverEnabled]: z.boolean(), + // Per-org sibling of snapshotStoreEverEnabled. One way: set when the org dial first moves past + // `off`, never cleared, so an org toggled dual-write -> off keeps probing its resident runs. + // System-set, so stripped from org payloads by withoutOrgForbiddenSnapshotKeys. Strict boolean. + [FEATURE_FLAG.snapshotStoreOrgEverEnabled]: z.boolean(), + // Global one-way "mode ever non-off" latch. Set true the first time the global dial is saved past + // `off`, never cleared. Distinct from snapshotStoreEverEnabled, the manual arming latch: this one + // means the dial WAS non-off. System-set, so stripped from org payloads. Strict boolean. + [FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled]: z.boolean(), + // Strict, like the other kill switches: a stringified "false" read as true would freeze every + // resident run's Redis head. + [FEATURE_FLAG.snapshotStoreHalt]: z.boolean(), }; export type FeatureFlagKey = keyof typeof FeatureFlagCatalog; @@ -181,6 +228,12 @@ export const GLOBAL_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.runOpsMintKindFlippedAt, FEATURE_FLAG.runOpsMintShardSetPrev, FEATURE_FLAG.runOpsMintShardSetFlippedAt, + // Read from the org blob only, and refused outright on a global save, so an editable control here + // would offer a setting whose only outcome is a 400. + FEATURE_FLAG.snapshotStoreOrgMode, + FEATURE_FLAG.snapshotStoreOrgEverEnabled, + // System-set global latch: read-only on the page, and the save path is its only writer. + FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled, ]; // Flags that are read-only on the org-level dialog. @@ -198,8 +251,99 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [ FEATURE_FLAG.runOpsMintShardSetPrev, FEATURE_FLAG.runOpsMintShardSetFlippedAt, FEATURE_FLAG.runOpsMintShardOverride, + // The dial and the hard stop are deployment-wide; only snapshotStoreOrgMode is per-org. + FEATURE_FLAG.snapshotStoreMode, + FEATURE_FLAG.snapshotStoreHalt, + FEATURE_FLAG.snapshotStoreEverEnabled, + // System-set latches: shown on the org dialog, but the operator never edits them. + FEATURE_FLAG.snapshotStoreOrgEverEnabled, + FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled, ]; +/** + * Drops keys an organisation must never supply. ORG_LOCKED_FLAGS is a UI predicate and no save path + * consults it, so the line is held here — the same way the mint grace stamps are stripped. + */ +export function withoutOrgForbiddenSnapshotKeys>(values: T): T { + const forbidden = [ + FEATURE_FLAG.snapshotStoreMode, + FEATURE_FLAG.snapshotStoreHalt, + // Deployment-wide, like the other two. Nothing reads it from an organisation row, so accepting + // it on an organisation save reports success for a setting that does nothing. + FEATURE_FLAG.snapshotStoreEverEnabled, + // System-set one-way latch. The save path is its only writer, so an operator-supplied value + // (a `false` above all) must never reach the stored blob. + FEATURE_FLAG.snapshotStoreOrgEverEnabled, + // Global system-set latch. Deployment-wide and written only by the global save path. + FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled, + ] as const; + if (!forbidden.some((key) => key in values)) return values; + + const rest = { ...values }; + for (const key of forbidden) { + delete rest[key]; + } + return rest; +} + +/** + * One-way per-org residency latch. Sets snapshotStoreOrgEverEnabled true when the resulting org dial + * is past `off`, and carries an already-set latch forward so a save back to `off` never clears it. + * Mutates and returns the stamped blob, which is written with replace semantics, so the carry-forward + * is what keeps the latch alive. Never writes `false`: an absent latch must stay distinguishable from + * an explicit one so the resolver keeps probing a resident org rather than skipping it. + */ +export function stampSnapshotStoreOrgEverEnabled( + existingFlags: Record | null | undefined, + stamped: Record +): Record { + const alreadyLatched = (existingFlags ?? {})[FEATURE_FLAG.snapshotStoreOrgEverEnabled] === true; + const mode = FeatureFlagCatalog[FEATURE_FLAG.snapshotStoreOrgMode].safeParse( + stamped[FEATURE_FLAG.snapshotStoreOrgMode] + ); + const enablingNow = mode.success && mode.data !== "off"; + + if (alreadyLatched || enablingNow) { + stamped[FEATURE_FLAG.snapshotStoreOrgEverEnabled] = true; + } + return stamped; +} + +/** + * The clear-all counterpart to the one-way per-org latch. Returns the blob to write when wiping an + * org's flags: `{ snapshotStoreOrgEverEnabled: true }` if the org was ever enabled (so it stays in + * the census), else null to wipe everything. Never resurrects a false or absent latch. + */ +export function clearedOrgFlagsPreservingLatch( + existingFlags: Record | null | undefined +): Record | null { + const latched = (existingFlags ?? {})[FEATURE_FLAG.snapshotStoreOrgEverEnabled] === true; + return latched ? { [FEATURE_FLAG.snapshotStoreOrgEverEnabled]: true } : null; +} + +/** + * One-way global latch. Sets snapshotStoreGlobalModeEverEnabled true when the resulting global dial + * is past `off`, and carries an already-set latch forward so a save back to `off` never clears it. + * The global sibling of stampSnapshotStoreOrgEverEnabled: same never-writes-false, carry-forward + * shape. Mutates and returns the stamped blob. + */ +export function stampSnapshotStoreGlobalModeEverEnabled( + existingFlags: Record | null | undefined, + stamped: Record +): Record { + const alreadyLatched = + (existingFlags ?? {})[FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled] === true; + const mode = FeatureFlagCatalog[FEATURE_FLAG.snapshotStoreMode].safeParse( + stamped[FEATURE_FLAG.snapshotStoreMode] + ); + const enablingNow = mode.success && mode.data !== "off"; + + if (alreadyLatched || enablingNow) { + stamped[FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled] = true; + } + return stamped; +} + /** * Flag groups where the operator sets a `primary` and the server computes the rest. The topology * lives here, not in the server module, because the admin page needs it too: unsetting a primary diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts index 76ac35f349b..30a5c549433 100644 --- a/apps/webapp/app/v3/runEngine.server.ts +++ b/apps/webapp/app/v3/runEngine.server.ts @@ -12,6 +12,8 @@ import { runEnginePendingVersionLookup } from "./runEnginePendingVersionLookup.s import { pickRunOpsStoreForCompletion } from "./runOpsMigration/crossSeamGuard.server"; import { runEngineControlPlaneResolver } from "./runOpsMigration/runEngineControlPlaneResolver.server"; import { runStore } from "./runStore.server"; +import { getSnapshotSweepRunner } from "./snapshotStoreBindings.server"; +import { getSnapshotStoreConfig } from "./snapshotStoreInstance.server"; import { meter, tracer } from "./tracer.server"; export const engine = singleton("RunEngine", createRunEngine); @@ -241,6 +243,22 @@ function createRunEngine() { randomize: true, }, }, + // Omitted entirely when the snapshot store is unconfigured: passing a runner would register the + // cron job and log an unbound pass every interval on every install that does not use the store. + snapshotStore: getSnapshotStoreConfig().configured + ? { + runSweep: async (opts) => { + const run = getSnapshotSweepRunner(); + if (!run) { + return { outcome: "unbound" }; + } + return run(opts); + }, + sweepSchedule: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_SCHEDULE, + sweepJitterInMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_JITTER_IN_MS, + sweepBudgetMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS, + } + : undefined, // Debounce configuration debounce: { maxDebounceDurationMs: env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS, diff --git a/apps/webapp/app/v3/runStore.server.ts b/apps/webapp/app/v3/runStore.server.ts index 8441f82ea1a..aae1bbde433 100644 --- a/apps/webapp/app/v3/runStore.server.ts +++ b/apps/webapp/app/v3/runStore.server.ts @@ -20,6 +20,9 @@ import { } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; +import { isSnapshotStoreConfigured } from "./snapshotStoreConfigured.server"; +import { decorateWithSnapshotStore } from "./snapshotStoreInstance.server"; +import { snapshotStoreModeResolver } from "./snapshotStoreMode.server"; import { resilienceForClient, type TransactionResilienceConfig, @@ -53,6 +56,10 @@ type BuildRunStoreDeps = { singleResilience?: TransactionResilienceConfig; newResilience?: TransactionResilienceConfig; legacyResilience?: TransactionResilienceConfig; + /** The redis-only PG-suppression predicate, or undefined for a plain passthrough. Passed in (never + * read from env here) so this builder stays pure; the caller wires it only when the store is + * configured, so an unconfigured deploy writes every snapshot row with no per-write resolution. */ + snapshotWrites?: (organizationId?: string) => boolean; }; /** @@ -60,13 +67,23 @@ type BuildRunStoreDeps = { * * Split OFF (default / self-host): returns the exact passthrough PostgresRunStore we * have always returned, built from the single control-plane handles. No second store - * is constructed and no marker predicate is consulted, so behavior is byte-identical - * to single-DB today. + * is constructed, and the redis-only marker predicate is consulted only when the caller + * wired one (i.e. the snapshot store is configured); with it unset behavior is + * byte-identical to single-DB today. * * Split ON: returns a RoutingRunStore that selects between a NEW store (where new runs * are born) and a LEGACY store (draining) by run-id residency (id shape). There is no cuid * migration, so a LEGACY-classified id is always LEGACY-resident. */ +// A run's org is redis-only exactly when the same resolver the snapshot decorator uses says so, so +// the Redis mirror and the Postgres suppression always agree on the effective mode for that run. +const suppressPgAtRedisOnly = (organizationId?: string) => + snapshotStoreModeResolver.resolve(organizationId) !== "redis-only"; + +// Wired into the store only when the snapshot store is configured. Unconfigured, this stays undefined +// and PostgresRunStore writes every snapshot row with no per-write mode resolution (the inert state). +const snapshotWritesPredicate = isSnapshotStoreConfigured() ? suppressPgAtRedisOnly : undefined; + export function buildRunStore(deps: BuildRunStoreDeps): RunStore { if (!deps.splitEnabled) { return new PostgresRunStore({ @@ -74,6 +91,7 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { readOnlyPrisma: deps.singleReplica, maxWait: deps.singleResilience?.maxWait, transactionStartRetry: deps.singleResilience?.startRetry, + snapshotWrites: deps.snapshotWrites, }); } @@ -89,12 +107,14 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { schemaVariant: "dedicated", maxWait: deps.newResilience?.maxWait, transactionStartRetry: deps.newResilience?.startRetry, + snapshotWrites: deps.snapshotWrites, }); const legacyStore = new PostgresRunStore({ prisma: deps.legacyWriter, readOnlyPrisma: deps.legacyReplica, maxWait: deps.legacyResilience?.maxWait, transactionStartRetry: deps.legacyResilience?.startRetry, + snapshotWrites: deps.snapshotWrites, }); // Gen-2 shards: one dedicated store per descriptor, handed to the N-way router. An aliased shard @@ -108,6 +128,7 @@ export function buildRunStore(deps: BuildRunStoreDeps): RunStore { schemaVariant: "dedicated" as const, maxWait: shard.resilience?.maxWait, transactionStartRetry: shard.resilience?.startRetry, + snapshotWrites: deps.snapshotWrites, }), aliasOf: shard.aliasOf, })); @@ -192,7 +213,12 @@ function tryResolveRunOpsHandles() { } } -export const runStore: RunStore = singleton("RunStore", () => { +/** + * The router with no snapshot decorator. One intended consumer: the orphan sweeper's rule-2 + * lookup, which must ask Postgres whether a run row exists and must never be able to ask Redis + * whether Redis is an orphan. Every other caller wants `runStore`. + */ +export const runStoreWithoutSnapshotDecorator: RunStore = singleton("RunStore.undecorated", () => { const handles = ROUTING_ENABLED ? tryResolveRunOpsHandles() : null; // Single-store passthrough: self-host (one DB), or a context without run-ops handles. if (!handles) { @@ -201,6 +227,7 @@ export const runStore: RunStore = singleton("RunStore", () => { singleWriter: prisma, singleReplica: $replica, singleResilience: resilienceForClient(prisma), + snapshotWrites: snapshotWritesPredicate, }); } const { shardHandles, ...storeHandles } = handles; @@ -219,5 +246,10 @@ export const runStore: RunStore = singleton("RunStore", () => { singleResilience: resilienceForClient(prisma), newResilience: resilienceForClient(handles.newWriter), legacyResilience: resilienceForClient(handles.legacyWriter), + snapshotWrites: snapshotWritesPredicate, }); }); + +export const runStore: RunStore = singleton("RunStore", () => + decorateWithSnapshotStore(runStoreWithoutSnapshotDecorator) +); diff --git a/apps/webapp/app/v3/snapshotRunOrg.server.ts b/apps/webapp/app/v3/snapshotRunOrg.server.ts new file mode 100644 index 00000000000..270386c6952 --- /dev/null +++ b/apps/webapp/app/v3/snapshotRunOrg.server.ts @@ -0,0 +1,118 @@ +import { LRUCache } from "lru-cache"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { singleton } from "~/utils/singleton"; + +/** + * How long an authoritative run→org read will wait before giving up. Shares the shape of the + * org-mode source's warm timeout, but here the deadline REJECTS: this feeds the redis-only fallback + * gate, which must fail closed rather than answer with a wrong org. + */ +const AUTHORITATIVE_TIMEOUT_MS = 500; + +const DEFAULT_CACHE_MAX = 50_000; + +/** + * What resolution needs from Prisma, narrowed so a test injects a hand-written fake rather than + * mocking the client. A run's organisation is reached through its environment: TaskRun.organizationId + * is nullable on historical rows, while RuntimeEnvironment.organizationId is not. + */ +type RunOrgClient = { + taskRun: { + findFirst(args: { + where: { id: string }; + select: { runtimeEnvironment: { select: { organizationId: true } } }; + }): Promise<{ runtimeEnvironment: { organizationId: string } } | null>; + }; +}; + +export type SnapshotRunOrgSource = { + /** + * Cache hit or undefined. On a miss, kicks off a replica populate off-path and returns at once. + * Never blocks, never throws. + */ + resolve(runId: string): string | undefined; + /** + * Awaits a bounded primary read, caches, and returns the org id. Throws on timeout, client + * failure, or a run with no organisation, so a caller can fail closed. + */ + resolveAuthoritative(runId: string): Promise; +}; + +export function createSnapshotRunOrgSource(clients?: { + primary: RunOrgClient; + replica: RunOrgClient; +}): SnapshotRunOrgSource { + const primaryClient = (clients?.primary ?? prisma) as RunOrgClient; + const replicaClient = (clients?.replica ?? $replica) as RunOrgClient; + // No ttl: run→org is immutable, so a cached mapping never goes stale. + const cache = new LRUCache({ + max: env.RUN_ENGINE_SNAPSHOT_STORE_RUN_ORG_CACHE_MAX ?? DEFAULT_CACHE_MAX, + }); + const inFlight = new Set(); + + async function read(runId: string, client: RunOrgClient): Promise { + return client.taskRun + .findFirst({ + where: { id: runId }, + select: { runtimeEnvironment: { select: { organizationId: true } } }, + }) + .then((row) => { + const organizationId = row?.runtimeEnvironment?.organizationId; + if (!organizationId) { + throw new Error(`snapshotRunOrg: no organization for run ${runId}`); + } + cache.set(runId, organizationId); + return organizationId; + }); + } + + return { + resolve(runId) { + const cached = cache.get(runId); + if (cached !== undefined) { + return cached; + } + if (!inFlight.has(runId)) { + inFlight.add(runId); + void read(runId, replicaClient) + .catch((error) => { + logger.warn("snapshotRunOrg: run→org populate failed", { runId, error }); + }) + .finally(() => { + inFlight.delete(runId); + }); + } + return undefined; + }, + async resolveAuthoritative(runId) { + const cached = cache.get(runId); + if (cached !== undefined) { + return cached; + } + + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error(`snapshotRunOrg: run→org read exceeded ${AUTHORITATIVE_TIMEOUT_MS}ms`) + ), + AUTHORITATIVE_TIMEOUT_MS + ); + }); + + try { + return await Promise.race([read(runId, primaryClient), deadline]); + } finally { + if (timer) clearTimeout(timer); + } + }, + }; +} + +/** Built on first use, never at import: importing this module must have no side effect. */ +export function snapshotRunOrgSource(): SnapshotRunOrgSource { + return singleton("snapshotRunOrgSource", () => createSnapshotRunOrgSource()); +} diff --git a/apps/webapp/app/v3/snapshotStoreBindings.server.ts b/apps/webapp/app/v3/snapshotStoreBindings.server.ts new file mode 100644 index 00000000000..e10bbe796b6 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreBindings.server.ts @@ -0,0 +1,31 @@ +import type { SnapshotRepairEnqueuer } from "@internal/run-store"; + +export type SweepPassOutcome = { + outcome: "completed" | "partial" | "skipped_locked" | "failed" | "unbound" | "aborted"; + counts?: Record; +}; + +export type SweepRunner = (opts: { + deadline: number; + signal: AbortSignal; +}) => Promise; + +/** Late-bound so the run store never has to import the engine. A third module wires both at boot. */ +let repairEnqueuer: SnapshotRepairEnqueuer | undefined; +let sweepRunner: SweepRunner | undefined; + +export function setSnapshotRepairEnqueuer(fn: SnapshotRepairEnqueuer): void { + repairEnqueuer = fn; +} + +export function getSnapshotRepairEnqueuer(): SnapshotRepairEnqueuer | undefined { + return repairEnqueuer; +} + +export function setSnapshotSweepRunner(fn: SweepRunner): void { + sweepRunner = fn; +} + +export function getSnapshotSweepRunner(): SweepRunner | undefined { + return sweepRunner; +} diff --git a/apps/webapp/app/v3/snapshotStoreBoot.server.ts b/apps/webapp/app/v3/snapshotStoreBoot.server.ts new file mode 100644 index 00000000000..e96f5f26f50 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreBoot.server.ts @@ -0,0 +1,228 @@ +import { scanTargetsOf, type SnapshotStoreMode } from "@internal/run-store"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; +import { getSnapshotRepairEnqueuer } from "./snapshotStoreBindings.server"; +import { getSnapshotStoreConfig, getSnapshotSweepClient } from "./snapshotStoreInstance.server"; + +/** + * Per node, because in cluster mode the policy is node-local: one replacement node brought up on a + * default config evicts the slots it owns while every other node stays safe. `unknown` covers a + * managed endpoint that refuses CONFIG GET, which is evidence of nothing either way. + */ +type EvictionPolicyReport = + | { kind: "unknown"; reason: string } + | { kind: "known"; nodes: { node: string; policy: string }[] }; + +export type SnapshotStoreBootDeps = { + mode: SnapshotStoreMode; + hostConfigured: boolean; + completedTtlMs: number; + orphanAgeMs: number; + ping: () => Promise; + evictionPolicy: () => Promise; + repairBound: () => boolean; + /** + * Whether the retired RUN_ENGINE_SNAPSHOT_STORE_HALT still asks for a halt. Only "1" counts: the + * old default was "0", which carries no intent, and refusing on it would stop every deployment + * that pins the default while asking for nothing. + */ + legacyEnvHalt: boolean; + log: (message: string, fields?: Record) => void; + warn: (message: string, fields?: Record) => void; +}; + +const PING_TIMEOUT_MS = 5_000; +const FLAG_READY_TIMEOUT_MS = 10_000; + +export async function assertSnapshotStoreBoot(deps: SnapshotStoreBootDeps): Promise { + // Checked before anything else, and it does not depend on the dial or the host. The variable no + // longer halts anything, so leaving it set means an operator believes the mirror is stopped while + // it is running. That is the one failure mode worse than not starting. + if (deps.legacyEnvHalt) { + throw new Error( + "RUN_ENGINE_SNAPSHOT_STORE_HALT is set but no longer does anything; the hard stop is the snapshotStoreHalt feature flag. Set the flag, then unset this variable." + ); + } + + const pastOff = deps.mode !== "off"; + // Configuration is validated as soon as a host is set, NOT only past off. The per-organisation + // override can put one organisation at dual-write while the deployment dial is still off, which is + // how a ramp starts, so keying these on the deployment dial let a ramped organisation run on a + // configuration nothing had checked. Reachability stays dial-gated below, because that one is a + // transient fault rather than bad config. + const configured = deps.hostConfigured; + + if (pastOff && !deps.hostConfigured) { + throw new Error( + `Snapshot store dial is "${deps.mode}" but RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST is unset, so nothing is constructed; refusing to start.` + ); + } + + if (configured && !(deps.completedTtlMs > 0)) { + throw new Error("RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS must be a positive integer."); + } + + if (configured && !(deps.orphanAgeMs > 0)) { + throw new Error("RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS must be a positive integer."); + } + + // Nothing enforces that a process which appends has imported the engine module, and an unbound + // enqueuer loses every repair job silently — which burns a task attempt per lost repair. + if (configured && !deps.repairBound()) { + throw new Error( + "Snapshot store dial is past off but the repair enqueuer is unbound; refusing to start." + ); + } + + if (pastOff) { + const reachable = await deps.ping(); + if (!reachable) { + if (deps.mode === "redis-only") { + throw new Error( + "Snapshot store dial is redis-only and the endpoint is unreachable; refusing to start." + ); + } + // Postgres is authoritative below redis-only, so a lost append costs nothing. Refusing here + // would bleed fleet capacity during a Redis fault to protect a write path that is free. + deps.warn("Snapshot store Redis is unreachable; booting because Postgres is authoritative", { + mode: deps.mode, + }); + } else { + await assertNoEviction(deps); + } + } + + if (deps.mode === "redis-only") { + deps.warn( + "Snapshot store dial is redis-only: Postgres snapshot writes are suppressed, so Redis is the sole store for run snapshots. A Redis fault at this position fails run creation and cannot be repaired from Postgres, and rolling the dial down does not recover runs born here.", + { mode: deps.mode } + ); + } + + deps.log("snapshot store resolved", { + mode: deps.mode, + hostConfigured: deps.hostConfigured, + completedTtlMs: deps.completedTtlMs, + orphanAgeMs: deps.orphanAgeMs, + }); +} + +/** + * Refuses at every dial past off, unlike the reachability check above. + * + * An unreachable endpoint is a transient fault that heals itself and costs nothing below + * redis-only, so refusing there would only bleed capacity. An evicting policy is static config + * that will not heal, and it removes the keys that make a run's keyspace live: writes freeze, + * reads fall back to Postgres for the rest of the run, and a birth landing after the eviction + * restarts the entry sequence beneath a surviving index. The dial is a runtime flag that can be + * raised to redis-read with no restart, so boot is the only place this is ever checked. + */ +async function assertNoEviction(deps: SnapshotStoreBootDeps): Promise { + const report = await deps.evictionPolicy(); + + if (report.kind === "unknown") { + deps.warn("Could not read the snapshot store maxmemory-policy; assuming noeviction", { + mode: deps.mode, + reason: report.reason, + }); + return; + } + + if (report.nodes.length === 0) { + deps.warn("No snapshot store node reported a maxmemory-policy", { mode: deps.mode }); + return; + } + + const evicting = report.nodes.filter((node) => node.policy !== "noeviction"); + if (evicting.length > 0) { + const described = evicting.map((node) => `${node.node}=${node.policy}`).join(", "); + throw new Error( + `Snapshot store dial is "${deps.mode}" but the endpoint may evict keys (${described}); ` + + "the mirror requires maxmemory-policy noeviction on every node. Refusing to start." + ); + } +} + +async function readEvictionPolicy(): Promise { + const client = getSnapshotSweepClient(); + if (!client) { + return { kind: "unknown", reason: "no snapshot store client" }; + } + try { + const nodes = await Promise.all( + scanTargetsOf(client).map(async (node) => { + const raw = (await node.config("GET", "maxmemory-policy")) as unknown; + return { + node: `${node.options.host ?? "unknown"}:${node.options.port ?? 0}`, + policy: policyFromConfigGet(raw), + }; + }) + ); + return { kind: "known", nodes }; + } catch (error) { + return { kind: "unknown", reason: error instanceof Error ? error.message : String(error) }; + } +} + +/** CONFIG GET answers as a flat [name, value] array on RESP2 and as a map on RESP3. */ +function policyFromConfigGet(raw: unknown): string { + if (Array.isArray(raw)) { + return String(raw[1] ?? ""); + } + if (raw && typeof raw === "object") { + return String((raw as Record)["maxmemory-policy"] ?? ""); + } + return ""; +} + +async function pingSweepClient(): Promise { + const client = getSnapshotSweepClient(); + if (!client) { + return false; + } + try { + const result = await Promise.race([ + client.ping(), + new Promise((_, reject) => + setTimeout(() => reject(new Error("ping timed out")), PING_TIMEOUT_MS) + ), + ]); + return result === "PONG"; + } catch { + return false; + } +} + +/** The env-reading adapter. The boot log line is not authoritative after boot: the dial can move. */ +export async function assertSnapshotStoreBootFromEnv(): Promise { + // Wait for the flag snapshot's first load. Without this the resolved dial is always the env + // floor, because a cold registry returns undefined, and the configuration check would only ever + // see a value no operator sets. A registry that never loads leaves the check on the floor, which + // fails toward inert. + await Promise.race([ + globalFlagsRegistry.isReady, + new Promise((resolve) => setTimeout(resolve, FLAG_READY_TIMEOUT_MS)), + ]); + + const config = getSnapshotStoreConfig(); + + await assertSnapshotStoreBoot({ + mode: config.mode, + hostConfigured: config.configured, + completedTtlMs: config.completedTtlMs, + orphanAgeMs: config.orphanAgeMs, + ping: pingSweepClient, + evictionPolicy: readEvictionPolicy, + repairBound: () => !!getSnapshotRepairEnqueuer(), + // Through the env adapter like every other variable. It is in the schema for this check alone. + legacyEnvHalt: env.RUN_ENGINE_SNAPSHOT_STORE_HALT === "1", + log: (message, fields) => + logger.info(message, { + ...fields, + keyPrefix: config.keyPrefix, + clusterMode: config.clusterMode, + }), + warn: (message, fields) => logger.warn(message, fields), + }); +} diff --git a/apps/webapp/app/v3/snapshotStoreConfigured.server.ts b/apps/webapp/app/v3/snapshotStoreConfigured.server.ts new file mode 100644 index 00000000000..46b0c6ef3a8 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreConfigured.server.ts @@ -0,0 +1,13 @@ +import { env } from "~/env.server"; + +/** + * The single bootstrap switch for the snapshot store. With no Redis host configured the feature is + * inert: the org census does not poll, the run store is a plain Postgres passthrough with no per-write + * mode resolution, and no decorator is attached. Every optional piece gates on this one predicate so + * that merging the feature with the host unset adds zero standing cost. + */ +export function isSnapshotStoreConfigured( + host: string | undefined = env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST ?? undefined +): boolean { + return !!host; +} diff --git a/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts b/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts new file mode 100644 index 00000000000..7d4bf30f1da --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreFlagGuard.server.ts @@ -0,0 +1,52 @@ +import { FEATURE_FLAG } from "~/v3/featureFlags"; + +/** + * Refuses a dial flip that would be silent: with no host the store is never constructed, so a flag + * past `off` has no effect. Cannot live in validatePartialFeatureFlags, which client components + * import and which therefore can never read env. + */ +export function snapshotStoreFlagSaveError( + requested: Record, + // Both REQUIRED. `everEnabled` was optional so as not to disturb existing callers, and a route + // that omitted it silently skipped the latch check: an optional safety argument disables the + // safety at every caller that forgets it. Required means the compiler enumerates them. + opts: { redisHostConfigured: boolean; everEnabled: boolean } +): string | undefined { + // Both keys, because either one past `off` is equally silent without a connection, and either one + // equally makes a run resident once there is one. + const enabling = ([FEATURE_FLAG.snapshotStoreMode, FEATURE_FLAG.snapshotStoreOrgMode] as const) + .map((key) => ({ key, value: requested[key] })) + .filter(({ value }) => typeof value === "string" && value !== "off"); + + if (!opts.redisHostConfigured) { + for (const { key, value } of enabling) { + return `Cannot set ${key} to "${String(value)}": RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST is not configured in this deployment, so the snapshot store is never constructed and the flag would have no effect.`; + } + return undefined; + } + + // The latch must already be set before anything can become resident. Transitions skip Redis + // entirely while it is unset, so a run born after the dial moved but before the latch landed would + // be resident with its transitions skipped, and its head would freeze while Postgres moved on. + // Refusing here makes that ordering impossible to get wrong rather than merely documented. + if (!opts.everEnabled) { + for (const { key, value } of enabling) { + return `Cannot set ${key} to "${String(value)}" before ${FEATURE_FLAG.snapshotStoreEverEnabled} is true. Set that flag first: until it is, transitions skip the store entirely, so a run born now would be resident with its transitions skipped and its head would freeze.`; + } + } + + return undefined; +} + +/** + * Refuses an organisation-only key on a global save. Nothing reads the global row for it, so a + * value saved there is inert, and an inert control an operator can set is worse than no control. + */ +export function globalOnlySnapshotStoreFlagError( + requested: Record +): string | undefined { + if (FEATURE_FLAG.snapshotStoreOrgMode in requested) { + return `${FEATURE_FLAG.snapshotStoreOrgMode} is per-organisation only; nothing reads it from the global flags, so setting it here would have no effect.`; + } + return undefined; +} diff --git a/apps/webapp/app/v3/snapshotStoreInstance.server.ts b/apps/webapp/app/v3/snapshotStoreInstance.server.ts new file mode 100644 index 00000000000..b8f98c047a6 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreInstance.server.ts @@ -0,0 +1,172 @@ +import { + createRedisClient, + createRedisClusterClient, + type RedisClient, + type RedisOptions, +} from "@internal/redis"; +import { + RedisSnapshotStore, + TaskRunExecutionSnapshotStore, + type RunStore, +} from "@internal/run-store"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { singleton } from "~/utils/singleton"; +import { getSnapshotRepairEnqueuer } from "./snapshotStoreBindings.server"; +import { isSnapshotStoreConfigured } from "./snapshotStoreConfigured.server"; +import { snapshotStoreHalted, snapshotStoreModeResolver } from "./snapshotStoreMode.server"; +import { createSnapshotStoreMetrics } from "./snapshotStoreMetrics.server"; +import { snapshotStoreOrgCensus } from "./snapshotStoreOrgCensus.server"; +import { meter } from "./tracer.server"; + +const KEY_PREFIX = "engine:"; + +function isConfigured(): boolean { + return isSnapshotStoreConfigured(); +} + +function redisOptions(): RedisOptions { + return { + keyPrefix: KEY_PREFIX, + host: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST ?? undefined, + port: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT ?? undefined, + username: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_USERNAME ?? undefined, + password: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PASSWORD ?? undefined, + enableAutoPipelining: true, + // A snapshot append sits on a request path, and a birth writes Redis before Postgres. With the + // offline queue on and no command timeout, an append issued while the endpoint is unreachable + // waits for a reconnect that may never come, so the trigger request hangs instead of falling + // back to Postgres. Both settings are local to this store: the shared defaults are used by + // every other Redis client in the app and are not ours to change. + enableOfflineQueue: false, + commandTimeout: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_COMMAND_TIMEOUT_MS, + ...(env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }; +} + +function isClusterMode(): boolean { + return env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_CLUSTER_MODE_ENABLED === "1"; +} + +function buildClient(name: string): RedisClient { + const options = redisOptions(); + const onError = (error: Error) => + logger.error(`snapshot store redis client error (${name})`, { error }); + + if (!isClusterMode()) { + return createRedisClient(options, { onError }); + } + + return createRedisClusterClient( + { + nodes: [ + { + host: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST, + port: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT, + }, + ], + redisOptions: options, + // Setting this on `options` alone is not enough: a Cluster keeps its own offline queue, and + // while it cannot refresh its slot cache it queues there and the request waits. + failFast: true, + }, + { onError } + ); +} + +type Instance = { + sweepClient: RedisClient; + hotPathClient: RedisClient; + redisSnapshotStore: RedisSnapshotStore; + decorate: (store: RunStore) => RunStore; +}; + +const instance = singleton("snapshotStoreInstance", () => { + if (!isConfigured()) { + return undefined; + } + + const metrics = createSnapshotStoreMetrics(meter, (organizationId) => + snapshotStoreOrgCensus.isCohortMember(organizationId) + ); + + // The sweep gets a connection of its own so a full scan of every master can never stall a + // transition append. It also backs the sweep's exclusion lock. + const sweepClient = buildClient("sweep"); + + const hotPathClient = buildClient("store"); + + const redisSnapshotStore = new RedisSnapshotStore({ + client: hotPathClient, + completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, + metrics: metrics.store, + }); + + return { + sweepClient, + hotPathClient, + redisSnapshotStore, + decorate: (store: RunStore) => + new TaskRunExecutionSnapshotStore(store, { + store: redisSnapshotStore, + modeResolver: snapshotStoreModeResolver, + halted: snapshotStoreHalted, + // Pinned: the field defaults to 0, which would mean no read ever reaches Redis. The + // organisation is the ramp unit, so there is no percentage to ramp. + readPercent: 100, + metrics: metrics.decorator, + onAppendFailure: async (args) => { + const enqueue = getSnapshotRepairEnqueuer(); + if (!enqueue) { + logger.error("snapshot repair enqueuer is unbound; repair job dropped", args); + return; + } + await enqueue(args); + }, + }), + }; +}); + +/** Returns the store verbatim when no snapshot-store Redis is configured. */ +export function decorateWithSnapshotStore(store: RunStore): RunStore { + return instance ? instance.decorate(store) : store; +} + +export function getSnapshotSweepClient(): RedisClient | undefined { + return instance?.sweepClient; +} + +export function getSnapshotStoreConfig() { + return { + configured: isConfigured(), + mode: snapshotStoreModeResolver.resolve(), + completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, + orphanAgeMs: env.RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS, + keyPrefix: KEY_PREFIX, + clusterMode: isClusterMode(), + // Reported so the boot line records how an append behaves when Redis is unreachable. + commandTimeoutMs: env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_COMMAND_TIMEOUT_MS, + offlineQueue: false, + }; +} + +const extraQuits: (() => Promise)[] = []; + +/** Lets the wiring module hand back a teardown for what it built, so this owns closing everything. */ +export function registerSnapshotStoreQuit(quit: () => Promise): void { + extraQuits.push(quit); +} + +export async function quitSnapshotStoreClients(): Promise { + if (!instance) { + return; + } + for (const quit of extraQuits) { + await quit().catch(() => undefined); + } + await instance.sweepClient.quit().catch(() => undefined); + // Last: an append in flight must still land. The store's own quit() returns early on a + // caller-supplied client, so closing the socket is ours. + await instance.redisSnapshotStore.quit().catch(() => undefined); + await instance.hotPathClient.quit().catch(() => undefined); +} diff --git a/apps/webapp/app/v3/snapshotStoreMetrics.server.ts b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts new file mode 100644 index 00000000000..3a72d15ee17 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreMetrics.server.ts @@ -0,0 +1,117 @@ +import type { Meter } from "@internal/tracing"; +import { APPEND_RESULT_OUTCOMES } from "@internal/run-store"; +import type { DecoratorMetrics, SnapshotStoreMetrics } from "@internal/run-store"; +import { cohortMetricLabel } from "./cohortMetricLabel.server"; + +// Metric attributes must be bounded: every one of these is a time series. The store and the +// decorator type their outcome strings loosely, so anything unrecognised collapses to "other" +// rather than minting a series. +const APPEND_OUTCOMES = ["written", "duplicate", "forked", "skippedNoKeyspace"] as const; +const APPEND_TTLS = ["none", "completion", "reapplied"] as const; +/** Derived from the store's own vocabulary, so an added outcome cannot silently become "other". */ +export const WRITE_OUTCOMES = APPEND_RESULT_OUTCOMES; +const READ_SOURCES = ["redis", "postgres"] as const; +const WRITE_SITES = [ + "createRun", + "createCancelledRun", + "completeAttemptSuccess", + "expireRun", + "expireParkedRun", + "rescheduleRun", + "lockRunToWorker", + "createExecutionSnapshot", + "runInTransaction", + // The repair's own writes. Without this they collapse to "other", so the one number that says + // whether the repair works is missing, and a repair racing a live transition is indistinguishable + // from a real divergence on the fork alert. + "repairRedisHead", +] as const; +const READ_METHODS = [ + "findLatestExecutionSnapshot", + "findExecutionSnapshot", + "findManyExecutionSnapshots", + "findSnapshotCompletedWaitpointIds", + "findSnapshotCompletedWaitpointIdsWithPresence", +] as const; +const SNAPSHOT_OPS = [ + "append", + "getById", + "getLatest", + "getSince", + "getSinceCreatedAt", + "getSnapshotWaitpointIds", +] as const; + +function bounded(value: string, allowed: readonly string[]): string { + return allowed.includes(value) ? value : "other"; +} + +/** + * Every instrument is created inside this function. At module scope they would register on every + * boot, including deployments with no snapshot-store Redis configured. + */ +/** Exported so the paging rule for a forked append is written against the real name. */ +export const SNAPSHOT_STORE_WRITE_TOTAL = "run_engine.snapshot_store.write_total"; + +// The soak cohort predicate. The injected `isCohortMember` decides the per-org label; the +// `() => false` default labels everyone "other", minting no series, when no predicate is injected. +export function createSnapshotStoreMetrics( + meter: Meter, + isCohortMember: (organizationId: string) => boolean = () => false +) { + // Two layers, two counters. Sharing one would count a single logical write twice and mix + // {outcome, ttl} points with {site, outcome} points under one name, so no sum or grouping over it + // would mean anything. + const appendTotal = meter.createCounter("run_engine.snapshot_store.append_total"); + const writeTotal = meter.createCounter(SNAPSHOT_STORE_WRITE_TOTAL); + const appendFailed = meter.createCounter("run_engine.snapshot_store.append_failed"); + const readSource = meter.createCounter("run_engine.snapshot_store.read_source"); + const skippedNoKeyspace = meter.createCounter("run_engine.snapshot_store.skipped_no_keyspace"); + const cycleMismatch = meter.createCounter("run_engine.snapshot_store.cycle_mismatch"); + const entryBytes = meter.createHistogram("run_engine.snapshot_store.entry_bytes"); + const cycleKeyBytes = meter.createHistogram("run_engine.snapshot_store.cycle_key_bytes"); + const cycleCount = meter.createHistogram("run_engine.snapshot_store.cycle_count"); + const opLatency = meter.createHistogram("run_engine.snapshot_store.op_latency_ms"); + + const store: SnapshotStoreMetrics = { + recordAppend: (outcome, ttl, organizationId) => + appendTotal.add(1, { + outcome: bounded(outcome, APPEND_OUTCOMES), + ttl: bounded(ttl, APPEND_TTLS), + org: cohortMetricLabel(organizationId, isCohortMember), + }), + recordEntryBytes: (bytes) => entryBytes.record(bytes), + recordCycleKeyBytes: (bytes) => cycleKeyBytes.record(bytes), + recordCycleCount: (count) => cycleCount.record(count), + recordSkippedNoKeyspace: () => skippedNoKeyspace.add(1), + recordCycleMismatch: () => cycleMismatch.add(1), + recordLatency: (op, ms) => opLatency.record(ms, { op: bounded(op, SNAPSHOT_OPS) }), + }; + + const decorator: DecoratorMetrics = { + recordWrite: (site, outcome) => { + writeTotal.add(1, { + site: bounded(site, WRITE_SITES), + outcome: bounded(outcome, WRITE_OUTCOMES), + }); + }, + recordAppendFailed: (site, organizationId) => + appendFailed.add(1, { + site: bounded(site, WRITE_SITES), + org: cohortMetricLabel(organizationId, isCohortMember), + }), + recordRead: (method, source) => + readSource.add(1, { + method: bounded(method, READ_METHODS), + // NOT `source`. Every exported series already carries a `source` label describing the + // telemetry pipeline, and a data point that repeats the name is dropped, so the whole + // metric was silently absent while the counter was being incremented. + served_by: bounded(source, READ_SOURCES), + }), + }; + + // No sweep emitter here. The engine owns the sweep metrics, because the engine is what schedules + // and runs a pass: see snapshotSweepPassCounter and snapshotSweepCountsHistogram. A second + // emitter on this side was dead, and its field list had already drifted from the engine's. + return { store, decorator }; +} diff --git a/apps/webapp/app/v3/snapshotStoreMode.server.ts b/apps/webapp/app/v3/snapshotStoreMode.server.ts new file mode 100644 index 00000000000..c485f5c1a45 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreMode.server.ts @@ -0,0 +1,386 @@ +import { LRUCache } from "lru-cache"; +import type { SnapshotStoreMode, SnapshotStoreModeResolver } from "@internal/run-store"; +import { $replica, prisma } from "~/db.server"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { singleton } from "~/utils/singleton"; +import { FEATURE_FLAG, FeatureFlagCatalog } from "~/v3/featureFlags"; +import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server"; +import { snapshotRunOrgSource } from "~/v3/snapshotRunOrg.server"; +import { snapshotStoreOrgCensus } from "~/v3/snapshotStoreOrgCensus.server"; + +/** A cached "this organisation has no override", distinct from "not cached". */ +export const NO_OVERRIDE = "__none__" as const; + +/** + * The dial positions, declared here rather than imported, so this module does not depend on the + * run-store package's build output to typecheck. The assertion below fails if the two ever diverge. + */ +type DialMode = "off" | "dual-write" | "redis-read" | "redis-only"; + +/** An organisation can be soaked at any ladder position, including the read positions. */ +type OrgDialMode = DialMode; + +type AssertSame = [A] extends [B] ? ([B] extends [A] ? true : never) : never; +const _dialMatchesRunStore: AssertSame = true; +void _dialMatchesRunStore; + +type CachedOrgMode = OrgDialMode | typeof NO_OVERRIDE; + +/** + * What to cache for one organisation's blob value. An unparseable or absent value caches + * NO_OVERRIDE rather than nothing: caching nothing means every organisation without an override + * re-queries on every write, which is all of them until a ramp starts. + */ +export function cachedOrgModeFor(raw: unknown): CachedOrgMode { + const parsed = FeatureFlagCatalog[FEATURE_FLAG.snapshotStoreOrgMode].safeParse(raw); + return parsed.success ? parsed.data : NO_OVERRIDE; +} + +/** One organisation's cached dial value. */ +type CachedOrg = { mode: CachedOrgMode }; + +type OrgModeSource = { + /** Cache-only: returns undefined on a true miss rather than querying. */ + get(organizationId: string): CachedOrgMode | undefined; + /** Fire-and-forget, de-duplicated per organisation, never throws. */ + refresh(organizationId: string): void; + /** Re-reads from the primary after a write, so replica lag cannot re-cache the old value. */ + invalidate(organizationId: string): void; + /** + * Awaits the organisation's value on a cache miss, bounded, and never throws. Used at BIRTH sites + * only: residency is permanent, so a run born during a miss is excluded from the mirror for life. + */ + warm(organizationId: string): Promise; +}; + +/** What resolution needs. Invalidation is a save-path concern, not a read-path one. */ +type ResolverOrgSource = Pick & + Partial>; + +/** Resolves a run to its organisation. Cache-only and synchronous, undefined on a miss. */ +type ResolverRunOrgSource = { + resolve(runId: string): string | undefined; + /** Bounded authoritative read, throws on failure/timeout, for the redis-only fallback gate. */ + resolveAuthoritative?(runId: string): Promise; +}; + +/** The census read accessors the resolver delegates to. Both synchronous and no-query. */ +type ResolverCensus = { + anyOrgReadEnabled(): boolean; + anyOrgRedisOnly(): boolean; +}; + +export function buildSnapshotStoreModeResolver(deps: { + globalMode: () => DialMode | undefined; + /** + * The one-way global-mode latch, cold-aware: true when the global dial has ever been non-off, and + * also true when the source is cold, so a cold read never permits a transition skip. Only a loaded + * source with the latch unset returns false. Absent is treated as true (never skip). + */ + globalModeEverEnabled?: () => boolean; + /** + * Whether an org is DEFINITELY never-enabled, per the census. Absent or cold means false, so an + * unknown answer keeps probing rather than suppressing a resident run. + */ + orgDefinitelyNeverEnabled?: (organizationId: string) => boolean; + orgMode: ResolverOrgSource; + /** Run→org resolution for the read path. Absent means readModeFor always falls back to global. */ + runOrg?: ResolverRunOrgSource; + /** The org census for the cheap read gates. Absent means both gates report false. */ + census?: ResolverCensus; + envFloor: DialMode; +}): SnapshotStoreModeResolver { + // Shared by resolve and readModeFor: the same org-mode logic, no read on this path. + const resolveMode = (organizationId?: string): DialMode => { + const global = deps.globalMode() ?? deps.envFloor; + if (!organizationId) { + return global; + } + + const cached = deps.orgMode.get(organizationId); + if (cached === NO_OVERRIDE) { + return global; + } + if (cached !== undefined) { + return cached; + } + + // Deliberately no read here. Seven decorator methods accept a caller-supplied `tx`, so a + // query on this path can land inside another caller's open interactive transaction, on the + // same pool for single-DB and self-host. Serve the global answer, warm the cache off-path. + try { + deps.orgMode.refresh(organizationId); + } catch { + // a warm-up must never fail a state transition + } + return global; + }; + + return { + // Cold-aware read delegated to deps; absent means true, so a transition never skips on a missing + // signal. The deps reader answers true while its source is cold. + globalModeEverEnabled: (): boolean => deps.globalModeEverEnabled?.() ?? true, + // False ONLY on a definite census negative; absent or cold means false, so an unknown answer + // keeps probing rather than suppressing a resident run. + orgDefinitelyNeverEnabled: (organizationId: string): boolean => + deps.orgDefinitelyNeverEnabled?.(organizationId) ?? false, + // Awaited at birth sites only. Absent org id means nothing to look up, so it is a no-op. + warm: async (organizationId: string): Promise => { + await deps.orgMode.warm?.(organizationId); + }, + resolve: (organizationId?: string): DialMode => resolveMode(organizationId), + // The org-scoped read position. Resolve run→org synchronously; on a miss return undefined so + // the decorator falls back to the global mode, which is safe during soak. + readModeFor: (runId: string): DialMode | undefined => { + const organizationId = deps.runOrg?.resolve(runId); + if (!organizationId) { + return undefined; + } + return resolveMode(organizationId); + }, + // Authoritative counterpart, used only when the sync read is unresolved and some org is + // redis-only. Resolves run→org from the primary (bounded, throws on failure), warms the org dial + // so the immediate read is accurate, then answers with the org's mode. A throw propagates so the + // decorator fails closed. + readModeForAuthoritative: async (runId: string): Promise => { + if (!deps.runOrg?.resolveAuthoritative) { + return undefined; + } + const organizationId = await deps.runOrg.resolveAuthoritative(runId); + await deps.orgMode.warm?.(organizationId); + return resolveMode(organizationId); + }, + anyOrgReadEnabled: (): boolean => deps.census?.anyOrgReadEnabled() ?? false, + anyOrgRedisOnly: (): boolean => deps.census?.anyOrgRedisOnly() ?? false, + }; +} + +/** + * The hard stop, and the flag is the whole of it. + * + * An environment half used to sit beside this, so a deployment could hold a halt the flag could not + * lift. It is gone. It converged over a rolling deploy rather than a flag interval, and for the + * length of that deploy the fleet is mixed: a halted process writes no transition, then an unhalted + * one asserts a head that was never written and forks. A control whose own convergence manufactures + * the divergence it exists to stop cannot be the way in. The guaranteed-inert state is an + * unconfigured host, which is bootstrap config and stays in the environment. + */ +export function buildSnapshotStoreHaltCheck(deps: { + flag: () => boolean | undefined; +}): () => boolean { + return () => deps.flag() === true; +} + +export const snapshotStoreHalted = buildSnapshotStoreHaltCheck({ + flag: () => globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreHalt], +}); + +/** + * How long a birth will wait for its organisation's dial before proceeding without it. Short because + * the read is a single primary-key select, and because the cost of overrunning is a caller's + * transaction held open. + */ +const WARM_TIMEOUT_MS = 500; + +const DEFAULT_CACHE_MAX = 10_000; +const DEFAULT_CACHE_TTL_MS = 30_000; + +// A failed post-save primary read must not wedge the org on the global fallback forever: retry the +// PRIMARY a few times (the replica stays blocked meanwhile so it cannot restore the superseded value), +// then give up and let normal refreshes resume. A bounded stale window beats a permanent one. +const DEFAULT_PRIMARY_INVALIDATE_RETRY_DELAYS_MS = [100, 250, 500, 1000]; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +type OrgModeClient = { + organization: { + findFirst(args: { + where: { id: string }; + select: { featureFlags: true }; + }): Promise<{ featureFlags: unknown } | null>; + }; +}; + +export function createOrgModeSource( + clients?: { + primary: OrgModeClient; + replica: OrgModeClient; + }, + opts?: { primaryInvalidateRetryDelaysMs?: number[] } +): OrgModeSource { + const primaryClient = (clients?.primary ?? prisma) as OrgModeClient; + const replicaClient = (clients?.replica ?? $replica) as OrgModeClient; + const primaryInvalidateRetryDelaysMs = + opts?.primaryInvalidateRetryDelaysMs ?? DEFAULT_PRIMARY_INVALIDATE_RETRY_DELAYS_MS; + // Defaults inline as well as in the schema: this must not throw when a caller supplies a partial + // env, and an LRU with neither bound set is a constructor error. + const cache = new LRUCache({ + max: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_MAX ?? DEFAULT_CACHE_MAX, + ttl: env.RUN_ENGINE_SNAPSHOT_STORE_ORG_MODE_CACHE_TTL_MS ?? DEFAULT_CACHE_TTL_MS, + }); + const inFlight = new Set(); + // Organisations whose primary read is still in flight after a save, keyed to the GENERATION that + // owns the read. A replica refresh started in that window would carry the SAME generation as the + // invalidation, so the generation guard below could not discard it, and a lagging replica landing + // second would restore the pre-save value for a full cache TTL. The save is the one read that must + // win, so nothing else reads during it. + // + // A Set was not enough. Two overlapping saves for one organisation share one entry, so the FIRST + // read's completion cleared it while the second was still outstanding, and the window reopened + // exactly when a second save made it most dangerous. Holding the generation means a completing + // read only clears the flag when it still owns it. + const primaryPending = new Map(); + // A replica read that started before an invalidation can land after the primary read and put the + // superseded value back. A per-organisation generation lets a stale load discard its own result. + const generations = new Map(); + const generationOf = (organizationId: string) => generations.get(organizationId) ?? 0; + + return { + get: (organizationId) => cache.get(organizationId)?.mode, + invalidate: (organizationId) => { + // Drop first, so a resolve between now and the re-read falls back rather than serving a + // value the write just replaced. + const generation = generationOf(organizationId) + 1; + generations.set(organizationId, generation); + cache.delete(organizationId); + primaryPending.set(organizationId, generation); + void loadWithPrimaryRetry(organizationId, generation); + }, + refresh: (organizationId) => { + // A save is mid-read for this organisation. Its answer is authoritative and a replica cannot + // improve on it, so skip: the resolver falls back to the global position until it lands. + if (primaryPending.has(organizationId) || inFlight.has(organizationId)) { + return; + } + inFlight.add(organizationId); + + void load(organizationId, replicaClient, generationOf(organizationId)).finally(() => { + inFlight.delete(organizationId); + }); + }, + warm: async (organizationId) => { + // Already known, including a cached "no override". Costs nothing, which is the common case + // once an organisation has any traffic at all. + if (cache.get(organizationId) !== undefined) { + return; + } + + // A save is mid-read: its answer is the authoritative one and is already on its way, so wait + // for that rather than starting a second read of the same row. + const pending = primaryPending.has(organizationId) + ? undefined + : load(organizationId, replicaClient, generationOf(organizationId)); + + // Bounded on purpose. A birth is on the trigger path and the caller may already hold an open + // transaction, so a slow flag read must give up rather than hold that transaction open. Giving + // up restores the previous behaviour (answer with the deployment-wide position) rather than + // failing the trigger. + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((resolve) => { + timer = setTimeout(resolve, WARM_TIMEOUT_MS); + }); + + try { + await Promise.race([pending ?? deadline, deadline]); + } finally { + if (timer) clearTimeout(timer); + } + }, + }; + + // The save's own primary read, with bounded retry. primaryPending stays set for `generation` across + // retries so a lagging replica cannot restore the superseded value; a success clears it, and so does + // exhausting the retries, so a persistent primary fault falls back to normal refreshes rather than + // wedging the org on the global position until the next save or a restart. + async function loadWithPrimaryRetry(organizationId: string, generation: number): Promise { + for (let attempt = 0; ; attempt++) { + const outcome = await load(organizationId, primaryClient, generation); + // A newer save (or its read) has superseded this one; it now owns the flag. + if (primaryPending.get(organizationId) !== generation) { + return; + } + if (outcome !== "failed") { + primaryPending.delete(organizationId); + return; + } + if (attempt >= primaryInvalidateRetryDelaysMs.length) { + // Retries exhausted: clear the flag so replica refreshes resume. The next save or reload + // corrects any briefly-restored stale value; a permanent wedge would not. + if (primaryPending.get(organizationId) === generation) { + primaryPending.delete(organizationId); + } + return; + } + await sleep(primaryInvalidateRetryDelaysMs[attempt]); + if (primaryPending.get(organizationId) !== generation) { + return; + } + } + } + + function load( + organizationId: string, + client: OrgModeClient, + generation: number + ): Promise<"loaded" | "stale" | "failed"> { + return client.organization + .findFirst({ where: { id: organizationId }, select: { featureFlags: true } }) + .then((row) => { + // Only the narrow per-org keys. The blob is never passed as `overrides` for the global + // key, where a parsing override would win outright. + const flags = row?.featureFlags as Record | null | undefined; + // A newer invalidation happened while this read was in flight, so its answer is stale. + if (generation < generationOf(organizationId)) { + return "stale" as const; + } + cache.set(organizationId, { + mode: cachedOrgModeFor(flags?.[FEATURE_FLAG.snapshotStoreOrgMode]), + }); + return "loaded" as const; + }) + .catch((error) => { + logger.warn("snapshotStoreMode: organisation override read failed", { + organizationId, + error, + }); + return "failed" as const; + }); + } +} + +/** Built on first use, never at import: importing this module must have no side effect. */ +function orgModeSource(): OrgModeSource { + return singleton("snapshotStoreOrgModeSource", createOrgModeSource); +} + +export const snapshotStoreModeResolver: SnapshotStoreModeResolver = buildSnapshotStoreModeResolver({ + globalMode: () => globalFlagsRegistry.current()?.[FEATURE_FLAG.snapshotStoreMode], + globalModeEverEnabled: () => { + // Conservative cold default: a cold registry must never permit a transition skip, so answer + // true. Only a loaded registry with the latch unset answers false (skip permitted). + const cur = globalFlagsRegistry.current(); + if (cur === undefined) return true; + return cur[FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled] === true; + }, + orgDefinitelyNeverEnabled: (organizationId) => + snapshotStoreOrgCensus.orgDefinitelyNeverEnabled(organizationId), + orgMode: { + get: (organizationId) => orgModeSource().get(organizationId), + refresh: (organizationId) => orgModeSource().refresh(organizationId), + warm: (organizationId) => orgModeSource().warm(organizationId), + }, + runOrg: { + resolve: (runId) => snapshotRunOrgSource().resolve(runId), + resolveAuthoritative: (runId) => snapshotRunOrgSource().resolveAuthoritative(runId), + }, + census: { + anyOrgReadEnabled: () => snapshotStoreOrgCensus.anyOrgReadEnabled(), + anyOrgRedisOnly: () => snapshotStoreOrgCensus.anyOrgRedisOnly(), + }, + envFloor: env.RUN_ENGINE_SNAPSHOT_STORE_MODE ?? "off", +}); + +/** Called by the organisation flag save path so the writing process sees a dial change at once. */ +export function invalidateSnapshotStoreOrgMode(organizationId: string): void { + orgModeSource().invalidate(organizationId); +} diff --git a/apps/webapp/app/v3/snapshotStoreOrgCensus.server.ts b/apps/webapp/app/v3/snapshotStoreOrgCensus.server.ts new file mode 100644 index 00000000000..b260a22e802 --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreOrgCensus.server.ts @@ -0,0 +1,138 @@ +import { Prisma } from "@trigger.dev/database"; +import { $replica } from "~/db.server"; +import { env } from "~/env.server"; +import { createReloadingRegistry } from "~/utils/reloadingRegistry.server"; +import { singleton } from "~/utils/singleton"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; +import { isSnapshotStoreConfigured } from "~/v3/snapshotStoreConfigured.server"; +import { cachedOrgModeFor, NO_OVERRIDE } from "~/v3/snapshotStoreMode.server"; + +/** The narrow slice of Prisma the census reads, so a test injects a fake without a mocking library. */ +export type SnapshotStoreOrgCensusClient = { + organization: { + findMany(args: { + where: { OR: Array<{ featureFlags: { path: string[]; not: typeof Prisma.DbNull } }> }; + select: { id: true; featureFlags: true }; + }): Promise>; + }; +}; + +/** Derived from the last successful load. */ +type OrgCensusSnapshot = { + /** Any org at redis-read or redis-only. */ + readEnabled: boolean; + /** Any org at redis-only. */ + redisOnly: boolean; + /** Orgs with any non-off override. */ + cohort: Set; + /** Orgs with the one-way per-org residency latch set true (may be off now, or latch-only). */ + everEnabled: Set; +}; + +export type SnapshotStoreOrgCensus = { + anyOrgReadEnabled(): boolean; + anyOrgRedisOnly(): boolean; + isCohortMember(organizationId: string): boolean; + /** DEFINITE never-enabled: census loaded AND the org is not in the ever-enabled set. */ + orgDefinitelyNeverEnabled(organizationId: string): boolean; + /** Force one load and await it. For tests and boot; the interval drives it in production. */ + refresh(): Promise; + stop(): void; +}; + +/** Classifies each org's blob exactly as the per-org resolver does (via cachedOrgModeFor). */ +function classify(rows: Array<{ id: string; featureFlags: unknown }>): OrgCensusSnapshot { + const cohort = new Set(); + const everEnabled = new Set(); + let readEnabled = false; + let redisOnly = false; + for (const row of rows) { + const flags = row.featureFlags as Record | null | undefined; + // Strict: only an explicit true latches, matching stampSnapshotStoreOrgEverEnabled. + if (flags?.[FEATURE_FLAG.snapshotStoreOrgEverEnabled] === true) everEnabled.add(row.id); + const mode = cachedOrgModeFor(flags?.[FEATURE_FLAG.snapshotStoreOrgMode]); + if (mode === NO_OVERRIDE || mode === "off") continue; + cohort.add(row.id); + if (mode === "redis-read" || mode === "redis-only") readEnabled = true; + if (mode === "redis-only") redisOnly = true; + } + return { readEnabled, redisOnly, cohort, everEnabled }; +} + +/** + * The census poll runs only when the store is configured AND we are outside test. The host gate is + * independent of NODE_ENV so a production process with no Redis host never starts the poll: the + * merged-but-off deploy pays no standing organization.findMany. + */ +export function defaultCensusAutoStart( + host: string | undefined = env.RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST ?? undefined, + nodeEnv: string | undefined = process.env.NODE_ENV +): boolean { + return isSnapshotStoreConfigured(host) && nodeEnv !== "test"; +} + +export function createSnapshotStoreOrgCensus( + clients?: { replica: SnapshotStoreOrgCensusClient }, + opts?: { intervalMs?: number; autoStart?: boolean } +): SnapshotStoreOrgCensus { + const client = (clients?.replica ?? $replica) as SnapshotStoreOrgCensusClient; + const registry = createReloadingRegistry({ + name: "snapshot-store-org-census", + intervalMs: opts?.intervalMs ?? env.GLOBAL_FLAGS_RELOAD_INTERVAL_MS, + autoStart: opts?.autoStart ?? defaultCensusAutoStart(), + load: async () => + // WHERE returns orgs with EITHER key, so an ever-enabled org that is now off (or holds only + // the latch after a clear) still returns. Classification stays in code, identical to the resolver. + classify( + await client.organization.findMany({ + where: { + OR: [ + { featureFlags: { path: [FEATURE_FLAG.snapshotStoreOrgMode], not: Prisma.DbNull } }, + { + featureFlags: { + path: [FEATURE_FLAG.snapshotStoreOrgEverEnabled], + not: Prisma.DbNull, + }, + }, + ], + }, + select: { id: true, featureFlags: true }, + }) + ), + }); + + return { + // Cold/failure fail-safe asymmetry. `current()` is undefined only before the first successful + // load; a later failure keeps the last-good snapshot, so these defaults apply to the cold window + // alone. The two read accessors err in OPPOSITE directions on purpose: + // - anyOrgReadEnabled -> TRUE: a false would suppress per-org read routing and silently break a + // soak org's reads. True only makes the decorator resolve the per-org mode, which is correct. + // - anyOrgRedisOnly -> FALSE: a true would trigger the conservative over-throw during the cold + // window; false lets reads fall back to Postgres, which is authoritative. Err toward fallback. + // - isCohortMember -> FALSE: cardinality-safe "other" for the metrics label until loaded. + anyOrgReadEnabled: () => registry.current()?.readEnabled ?? true, + anyOrgRedisOnly: () => registry.current()?.redisOnly ?? false, + isCohortMember: (organizationId) => registry.current()?.cohort.has(organizationId) ?? false, + // FALSE cold/never-loaded: not-definite means the caller (10c) must NOT skip. Only a loaded + // census whose ever-enabled set excludes the org yields a definite "never enabled". + orgDefinitelyNeverEnabled: (organizationId) => { + const snapshot = registry.current(); + return snapshot ? !snapshot.everEnabled.has(organizationId) : false; + }, + refresh: async () => { + // A failed reload keeps the last-good snapshot; swallow so accessors stay safe. + try { + await registry.reload(); + } catch {} + }, + stop: () => registry.stop(), + }; +} + +/** + * Built at import, like globalFlagsRegistry, but the poll starts only when a Redis host is configured + * (see defaultCensusAutoStart). Unconfigured, the object is inert: constructed, never polling. + */ +export const snapshotStoreOrgCensus = singleton("snapshotStoreOrgCensus", () => + createSnapshotStoreOrgCensus() +); diff --git a/apps/webapp/app/v3/snapshotStoreWiring.server.ts b/apps/webapp/app/v3/snapshotStoreWiring.server.ts new file mode 100644 index 00000000000..9cf96b0e35c --- /dev/null +++ b/apps/webapp/app/v3/snapshotStoreWiring.server.ts @@ -0,0 +1,63 @@ +import { SnapshotOrphanSweeper } from "@internal/run-store"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { signalsEmitter } from "~/services/signals.server"; +import { engine } from "./runEngine.server"; +import { runStoreWithoutSnapshotDecorator } from "./runStore.server"; +import { buildSnapshotSweepRunner } from "./snapshotSweepRunner.server"; +import { setSnapshotRepairEnqueuer, setSnapshotSweepRunner } from "./snapshotStoreBindings.server"; +import { + getSnapshotSweepClient, + quitSnapshotStoreClients, + registerSnapshotStoreQuit, +} from "./snapshotStoreInstance.server"; + +/** + * The third module: it imports both sides, so neither the run store nor the engine has to import + * the other. Invoked from entry.server.tsx, beside the other boot registrations. + */ +export function registerSnapshotStoreWiring(): boolean { + const sweepClient = getSnapshotSweepClient(); + + if (!sweepClient) { + return false; + } + + setSnapshotRepairEnqueuer(async (args) => { + await engine.enqueueSnapshotRepair(args); + }); + + const sweeper = new SnapshotOrphanSweeper({ + // Its own connection, so a scan of every master can never stall a transition append. + client: sweepClient, + // The undecorated router: rule 2 asks Postgres whether a run row exists, and must never be + // able to ask Redis whether Redis is an orphan. + runStore: runStoreWithoutSnapshotDecorator, + completedTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS, + orphanAgeMs: env.RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS, + confirmOrphanAfterMs: env.RUN_ENGINE_SNAPSHOT_STORE_CONFIRM_ORPHAN_AFTER_MS, + }); + + setSnapshotSweepRunner( + buildSnapshotSweepRunner({ + client: sweepClient, + sweep: async ({ deadline, signal }) => ({ ...(await sweeper.sweep({ deadline, signal })) }), + lockTtlMs: env.RUN_ENGINE_SNAPSHOT_STORE_GC_SWEEP_BUDGET_MS + 3_600_000, + }) + ); + + registerSnapshotStoreQuit(() => sweeper.quit()); + + // Close the sweeper and all three connections on the way out, the same way the other Redis-backed + // singletons do. `quitSnapshotStoreClients` is async and the signals emitter swallows listener + // rejections, so discard the promise explicitly rather than handing it a floating one. The caller + // wraps this function in `singleton`, so the listeners are registered once per process. + const onShutdown = (): void => { + void quitSnapshotStoreClients(); + }; + signalsEmitter.on("SIGTERM", onShutdown); + signalsEmitter.on("SIGINT", onShutdown); + + logger.info("snapshot store wiring registered"); + return true; +} diff --git a/apps/webapp/app/v3/snapshotSweepRunner.server.ts b/apps/webapp/app/v3/snapshotSweepRunner.server.ts new file mode 100644 index 00000000000..bf4150385e5 --- /dev/null +++ b/apps/webapp/app/v3/snapshotSweepRunner.server.ts @@ -0,0 +1,94 @@ +import type { RedisClient } from "@internal/redis"; +import { logger } from "~/services/logger.server"; +import type { SweepPassOutcome, SweepRunner } from "./snapshotStoreBindings.server"; + +const LOCK_KEY = "snapshot-sweep:lock"; + +// Compare-and-delete. A bare DEL would let a pass that overran its own lock delete its SUCCESSOR's +// lock on release, and two passes would then run together — the failure the lock exists to prevent. +const RELEASE_LUA = ` +if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) +end +return 0 +`; + +// The release is a WRITE, so the fault that fails a pass fails the release too; a single attempt +// then leaks the lock to its full TTL (budget + 2h), stalling every sweep for hours after a blip. +// Retry over a short window so a Redis that recovers frees the lock promptly; the TTL is the backstop. +const DEFAULT_RELEASE_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000]; + +const sleep = (ms: number) => + ms > 0 ? new Promise((resolve) => setTimeout(resolve, ms)) : Promise.resolve(); + +type SweepCounts = Record; + +async function releaseLock( + client: RedisClient, + fence: string, + retryDelaysMs: number[] +): Promise { + // attempts = retryDelaysMs.length + 1. A successful eval ends it, whether it deleted our lock + // (returned 1) or found the fence no longer ours (returned 0) — both mean nothing more to do. + for (let attempt = 0; ; attempt++) { + try { + await client.eval(RELEASE_LUA, 1, LOCK_KEY, fence); + return; + } catch (error) { + if (attempt >= retryDelaysMs.length) { + logger.warn( + "snapshot sweep lock release failed after retries; lock will clear on its TTL", + { error } + ); + return; + } + await sleep(retryDelaysMs[attempt]); + } + } +} + +export function buildSnapshotSweepRunner(deps: { + client: RedisClient; + sweep: (opts: { deadline: number; signal: AbortSignal }) => Promise; + lockTtlMs: number; + fence?: () => string; + /** Backoff between release attempts. Defaults to a bounded ~16s ramp; tests pass short delays. */ + releaseRetryDelaysMs?: number[]; +}): SweepRunner { + return async ({ deadline, signal }): Promise => { + // enqueueOnce gives no overlap protection: its dedup record IS the queue item and the ack + // deletes it, so it elects a winner only at start-up. Nothing extends the visibility timeout + // either, so a long pass is redelivered and would otherwise run beside itself. + const fence = + deps.fence?.() ?? `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`; + const acquired = await deps.client.set(LOCK_KEY, fence, "PX", deps.lockTtlMs, "NX"); + + if (acquired !== "OK") { + return { outcome: "skipped_locked" }; + } + + try { + const counts = await deps.sweep({ deadline, signal }); + + if (signal.aborted) { + return { outcome: "aborted", counts }; + } + if (counts.partial === true) { + return { outcome: "partial", counts }; + } + return { outcome: "completed", counts }; + } catch (error) { + if (signal.aborted) { + return { outcome: "aborted" }; + } + logger.error("snapshot orphan sweep pass failed", { error }); + return { outcome: "failed" }; + } finally { + await releaseLock( + deps.client, + fence, + deps.releaseRetryDelaysMs ?? DEFAULT_RELEASE_RETRY_DELAYS_MS + ); + } + }; +} diff --git a/apps/webapp/test/adminOrgFeatureFlagsV1Route.test.ts b/apps/webapp/test/adminOrgFeatureFlagsV1Route.test.ts new file mode 100644 index 00000000000..f8568df36f7 --- /dev/null +++ b/apps/webapp/test/adminOrgFeatureFlagsV1Route.test.ts @@ -0,0 +1,129 @@ +// The v1 PAT route enables an org's snapshot dial with merge semantics. It must stamp the one-way +// per-org residency latch (snapshotStoreOrgEverEnabled) exactly as the v2 route does; without it the +// census keeps the org classified definitely-never-enabled and its resident runs' transitions are +// skipped. These drive the real exported action against a real Postgres and assert the stored blob. +// The guard is stubbed to a pass so the test isolates the stamp wiring (the guard has its own tests); +// only peripheral module boundaries are substituted, the database is the genuine article. +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { FEATURE_FLAG } from "~/v3/featureFlags"; + +vi.setConfig({ testTimeout: 60_000 }); + +const db = vi.hoisted(() => ({ client: null as unknown as PrismaClient })); + +vi.mock("~/services/personalAccessToken.server", () => ({ + requireAdminApiRequest: async () => ({}), +})); + +vi.mock("~/db.server", () => ({ + get prisma() { + return db.client; + }, + get $replica() { + return db.client; + }, +})); + +// Bypass the host/arming-latch gate so the test exercises the stamp path directly. The guard's own +// behaviour (reject without a host or before the global latch) is covered in its own tests. +vi.mock("~/v3/snapshotStoreFlagGuard.server", () => ({ + snapshotStoreFlagSaveError: () => undefined, +})); + +vi.mock("~/v3/runOpsMigration/controlPlaneResolver.server", () => ({ + controlPlaneResolver: { invalidateOrganization: () => {} }, +})); + +// Only used to seed the mint-flip baseline; irrelevant to the residency latch under test. +vi.mock("~/v3/featureFlags.server", () => ({ + flags: async () => ({}), +})); + +import { action } from "~/routes/admin.api.v1.orgs.$organizationId.feature-flags"; + +const MODE = FEATURE_FLAG.snapshotStoreOrgMode; +const LATCH = FEATURE_FLAG.snapshotStoreOrgEverEnabled; + +let orgSeq = 0; + +async function seedOrg(prisma: PrismaClient, featureFlags?: Record) { + db.client = prisma; + const id = `org_v1route_${orgSeq++}`; + await prisma.organization.create({ + data: { + id, + title: "V1 route test org", + slug: `v1-route-${id}`, + ...(featureFlags ? { featureFlags } : {}), + }, + }); + return id; +} + +async function post(organizationId: string, body: unknown) { + const request = new Request( + `https://localhost:3030/admin/api/v1/orgs/${organizationId}/feature-flags`, + { + method: "POST", + body: JSON.stringify(body), + headers: { "content-type": "application/json" }, + } + ); + return (await (action as any)({ + request, + params: { organizationId }, + context: {}, + })) as Response; +} + +async function readFlags(prisma: PrismaClient, id: string) { + const row = await prisma.organization.findFirst({ + where: { id }, + select: { featureFlags: true }, + }); + return (row?.featureFlags ?? null) as Record | null; +} + +describe("admin v1 org feature-flags route stamps the per-org residency latch", () => { + postgresTest("stamps the latch when the dial is enabled past off", async ({ prisma }) => { + const id = await seedOrg(prisma); + + const response = await post(id, { [MODE]: "redis-read" }); + + expect(response.status).toBe(200); + const flags = await readFlags(prisma, id); + expect(flags?.[MODE]).toBe("redis-read"); + expect(flags?.[LATCH]).toBe(true); + }); + + postgresTest( + "keeps the latch when a later save sets the dial back to off", + async ({ prisma }) => { + const id = await seedOrg(prisma); + + await post(id, { [MODE]: "redis-read" }); + const response = await post(id, { [MODE]: "off" }); + + expect(response.status).toBe(200); + const flags = await readFlags(prisma, id); + expect(flags?.[MODE]).toBe("off"); + // One-way: the latch survives the roll-back so a run still resident keeps mirroring. + expect(flags?.[LATCH]).toBe(true); + } + ); + + postgresTest( + "does not stamp the latch for an off save with no prior latch", + async ({ prisma }) => { + const id = await seedOrg(prisma); + + const response = await post(id, { [MODE]: "off" }); + + expect(response.status).toBe(200); + const flags = await readFlags(prisma, id); + expect(flags?.[LATCH]).toBeUndefined(); + } + ); +}); diff --git a/apps/webapp/test/cohortMetricLabel.test.ts b/apps/webapp/test/cohortMetricLabel.test.ts new file mode 100644 index 00000000000..09c24611346 --- /dev/null +++ b/apps/webapp/test/cohortMetricLabel.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { cohortMetricLabel } from "~/v3/cohortMetricLabel.server"; + +describe("cohortMetricLabel", () => { + it("returns the org id when it is a cohort member", () => { + expect(cohortMetricLabel("org_123", (id) => id === "org_123")).toBe("org_123"); + }); + + it("returns 'other' for a non-member org", () => { + expect(cohortMetricLabel("org_999", (id) => id === "org_123")).toBe("other"); + }); + + it("returns 'other' when the org id is undefined", () => { + expect(cohortMetricLabel(undefined, () => true)).toBe("other"); + }); + + it("collapses every org to 'other' with an always-false predicate", () => { + for (const id of ["org_a", "org_b", "org_c"]) { + expect(cohortMetricLabel(id, () => false)).toBe("other"); + } + }); +}); diff --git a/apps/webapp/test/featureFlags.test.ts b/apps/webapp/test/featureFlags.test.ts index 56ba4625292..6d6508653e8 100644 --- a/apps/webapp/test/featureFlags.test.ts +++ b/apps/webapp/test/featureFlags.test.ts @@ -6,7 +6,11 @@ import type { PrismaClient } from "@trigger.dev/database"; import { postgresTest } from "@internal/testcontainers"; import { describe, expect, it, vi } from "vitest"; import type { PrismaClientOrTransaction } from "~/db.server"; -import { FEATURE_FLAG, hasUnreadableTurnEvalsOverride } from "~/v3/featureFlags"; +import { + FEATURE_FLAG, + hasUnreadableTurnEvalsOverride, + validatePartialFeatureFlags, +} from "~/v3/featureFlags"; import { makeFlag, makeSetFlag } from "~/v3/featureFlags.server"; vi.setConfig({ testTimeout: 60_000 }); @@ -94,6 +98,23 @@ describe("flag() override resolution", () => { }); }); +// The per-org snapshot dial ladders through the same positions as the global dial, so an org +// can be soaked at redis-read and redis-only, not just off and dual-write. +describe("snapshotStoreOrgMode ladder on the org save path", () => { + const ORG_KEY = FEATURE_FLAG.snapshotStoreOrgMode; + + it("accepts every ladder position an org may be soaked at", () => { + for (const value of ["off", "dual-write", "redis-read", "redis-only"]) { + const parsed = validatePartialFeatureFlags({ [ORG_KEY]: value }); + expect(parsed.success, value).toBe(true); + } + }); + + it("still rejects a bogus position", () => { + expect(validatePartialFeatureFlags({ [ORG_KEY]: "redis-write" }).success).toBe(false); + }); +}); + // The fall-through above is right for an entitlement and wrong for consent: judging sends the // turn to a third-party model, so the eval flag refuses on an override it cannot read. describe("hasUnreadableTurnEvalsOverride", () => { diff --git a/apps/webapp/test/snapshotRunOrg.server.test.ts b/apps/webapp/test/snapshotRunOrg.server.test.ts new file mode 100644 index 00000000000..e405d0c1722 --- /dev/null +++ b/apps/webapp/test/snapshotRunOrg.server.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; +import { createSnapshotRunOrgSource } from "~/v3/snapshotRunOrg.server"; + +type Row = { runtimeEnvironment: { organizationId: string } } | null; + +/** A hand-written fake standing in for the Prisma client, so no mocking is needed. */ +function fakeClient(opts: { + mapping?: Record; + reject?: boolean; + delayMs?: number; +}) { + let calls = 0; + return { + get calls() { + return calls; + }, + taskRun: { + async findFirst(args: { where: { id: string } }): Promise { + calls++; + if (opts.delayMs) { + await new Promise((resolve) => setTimeout(resolve, opts.delayMs)); + } + if (opts.reject) { + throw new Error("db unreachable"); + } + const organizationId = opts.mapping?.[args.where.id]; + return organizationId ? { runtimeEnvironment: { organizationId } } : null; + }, + }, + }; +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 10)); + +describe("snapshot run→org source", () => { + it("returns undefined on a cold miss, then serves the org id once the populate settles", async () => { + const replica = fakeClient({ mapping: { run_a: "org_a" } }); + const source = createSnapshotRunOrgSource({ primary: replica, replica }); + + expect(source.resolve("run_a")).toBeUndefined(); + + await tick(); + + expect(source.resolve("run_a")).toBe("org_a"); + }); + + it("does not start a second populate while one is in flight", async () => { + const replica = fakeClient({ mapping: { run_a: "org_a" }, delayMs: 20 }); + const source = createSnapshotRunOrgSource({ primary: replica, replica }); + + source.resolve("run_a"); + source.resolve("run_a"); + source.resolve("run_a"); + + await new Promise((resolve) => setTimeout(resolve, 40)); + + expect(replica.calls).toBe(1); + expect(source.resolve("run_a")).toBe("org_a"); + }); + + it("resolveAuthoritative returns the org id on success and caches it", async () => { + const primary = fakeClient({ mapping: { run_a: "org_a" } }); + const source = createSnapshotRunOrgSource({ primary, replica: primary }); + + await expect(source.resolveAuthoritative("run_a")).resolves.toBe("org_a"); + expect(source.resolve("run_a")).toBe("org_a"); + }); + + it("resolveAuthoritative throws when the run has no organization", async () => { + const primary = fakeClient({ mapping: {} }); + const source = createSnapshotRunOrgSource({ primary, replica: primary }); + + await expect(source.resolveAuthoritative("run_missing")).rejects.toThrow(); + }); + + it("resolveAuthoritative throws when the client rejects", async () => { + const primary = fakeClient({ reject: true }); + const source = createSnapshotRunOrgSource({ primary, replica: primary }); + + await expect(source.resolveAuthoritative("run_a")).rejects.toThrow(); + }); + + it("resolve stays silent and releases in-flight when findFirst throws synchronously", async () => { + let calls = 0; + const throwing = { + taskRun: { + findFirst() { + calls++; + throw new Error("sync boom"); + }, + }, + } as unknown as NonNullable[0]>["replica"]; + const source = createSnapshotRunOrgSource({ primary: throwing, replica: throwing }); + + expect(() => source.resolve("run_a")).not.toThrow(); + + await tick(); + + // In-flight was released, so a fresh miss starts a new populate rather than wedging forever. + expect(() => source.resolve("run_a")).not.toThrow(); + expect(calls).toBe(2); + }); + + it("resolveAuthoritative throws when the read exceeds the deadline", async () => { + const primary = fakeClient({ mapping: { run_a: "org_a" }, delayMs: 2000 }); + const source = createSnapshotRunOrgSource({ primary, replica: primary }); + + await expect(source.resolveAuthoritative("run_a")).rejects.toThrow(/deadline|exceed/i); + }); +}); diff --git a/apps/webapp/test/snapshotStoreAlerts.test.ts b/apps/webapp/test/snapshotStoreAlerts.test.ts new file mode 100644 index 00000000000..18e9875689c --- /dev/null +++ b/apps/webapp/test/snapshotStoreAlerts.test.ts @@ -0,0 +1,52 @@ +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { APPEND_RESULT_OUTCOMES } from "@internal/run-store"; +import { SNAPSHOT_STORE_WRITE_TOTAL } from "~/v3/snapshotStoreMetrics.server"; + +const RULES = fs.readFileSync( + path.resolve(__dirname, "../../../docker/config/alerts/snapshot-store.yml"), + "utf8" +); + +function ruleBlock(alertName: string): string { + const start = RULES.indexOf(`- alert: ${alertName}`); + if (start === -1) return ""; + const next = RULES.indexOf("- alert:", start + 1); + return RULES.slice(start, next === -1 ? undefined : next); +} + +describe("the snapshot store alerting rules", () => { + it("match on the name suffix, because the exported prefix is deployment-dependent", () => { + const matchers = [...RULES.matchAll(/__name__=~"([^"]+)"/g)].map((m) => m[1]); + + expect(matchers.length).toBeGreaterThan(0); + for (const matcher of matchers) { + expect(matcher.startsWith(".*")).toBe(true); + } + }); + + it("pages on a forked append, against the counter the decorator actually writes", () => { + const block = ruleBlock("SnapshotStoreAppendForked"); + const exported = SNAPSHOT_STORE_WRITE_TOTAL.replaceAll(".", "_"); + + expect(APPEND_RESULT_OUTCOMES).toContain("forked"); + // The name still comes from the code constant, which is the point of this test: a rule matching + // a metric nothing emits is the failure mode here, and that has happened once already. + // + // The trailing `(_total)?` is deliberate. OTel exporters may or may not append `_total` to a + // counter, and a matcher that guesses wrong matches nothing at all while looking perfectly + // reasonable. Accepting both spellings costs nothing; a silent alert costs everything. + expect(block).toContain(`{__name__=~".*${exported}(_total)?",outcome="forked"}`); + expect(block).toContain("severity: page"); + }); + + it("tolerates an optional _total on every matcher, so no rule can silently match nothing", () => { + const matchers = [...RULES.matchAll(/__name__=~"([^"]+)"/g)].map((m) => m[1]); + + expect(matchers.length).toBeGreaterThan(0); + for (const matcher of matchers) { + expect(matcher.endsWith("(_total)?")).toBe(true); + } + }); +}); diff --git a/apps/webapp/test/snapshotStoreBoot.test.ts b/apps/webapp/test/snapshotStoreBoot.test.ts new file mode 100644 index 00000000000..d10f578eca4 --- /dev/null +++ b/apps/webapp/test/snapshotStoreBoot.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, it } from "vitest"; +import { assertSnapshotStoreBoot } from "~/v3/snapshotStoreBoot.server"; + +type Recorded = { + logs: string[]; + warnings: string[]; + pings: number; + policyProbes: number; +}; + +function deps(overrides: Partial[0]> = {}) { + const recorded: Recorded = { logs: [], warnings: [], pings: 0, policyProbes: 0 }; + const base = { + mode: "off" as const, + hostConfigured: false, + completedTtlMs: 1, + orphanAgeMs: 1, + ping: async () => { + recorded.pings += 1; + return true; + }, + repairBound: () => true, + evictionPolicy: async () => { + recorded.policyProbes += 1; + return { kind: "known" as const, nodes: [{ node: "primary", policy: "noeviction" }] }; + }, + log: (message: string) => recorded.logs.push(message), + warn: (message: string) => recorded.warnings.push(message), + }; + return { ...base, ...overrides, recorded }; +} + +describe("assertSnapshotStoreBoot", () => { + it("passes and logs when the dial is off and nothing is configured", async () => { + const d = deps(); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.logs).toHaveLength(1); + }); + + it("does not probe reachability at off", async () => { + const d = deps(); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.pings).toBe(0); + }); + + it("refuses a dial past off with no host", async () => { + await expect( + assertSnapshotStoreBoot(deps({ mode: "dual-write", hostConfigured: false })) + ).rejects.toThrow(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("refuses a non-positive TTL once the dial is past off", async () => { + await expect( + assertSnapshotStoreBoot(deps({ mode: "dual-write", hostConfigured: true, completedTtlMs: 0 })) + ).rejects.toThrow(/COMPLETED_TTL_MS/); + await expect( + assertSnapshotStoreBoot(deps({ mode: "dual-write", hostConfigured: true, orphanAgeMs: -1 })) + ).rejects.toThrow(/ORPHAN_AGE_MS/); + }); + + it("refuses when the repair binding is unset and the dial is past off", async () => { + await expect( + assertSnapshotStoreBoot( + deps({ mode: "dual-write", hostConfigured: true, repairBound: () => false }) + ) + ).rejects.toThrow(/repair/i); + }); + + it("boots on an unreachable endpoint below redis-only, loudly", async () => { + const d = deps({ mode: "dual-write", hostConfigured: true, ping: async () => false }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.warnings.length).toBeGreaterThan(0); + }); + + it("refuses an unreachable endpoint at redis-only", async () => { + await expect( + assertSnapshotStoreBoot( + deps({ mode: "redis-only", hostConfigured: true, ping: async () => false }) + ) + ).rejects.toThrow(/unreachable/i); + }); + + it("refuses a dial past off when the endpoint evicts keys", async () => { + await expect( + assertSnapshotStoreBoot( + deps({ + mode: "dual-write", + hostConfigured: true, + evictionPolicy: async () => ({ + kind: "known", + nodes: [{ node: "primary", policy: "allkeys-lru" }], + }), + }) + ) + ).rejects.toThrow(/noeviction/i); + }); + + it("refuses when any one cluster node evicts", async () => { + await expect( + assertSnapshotStoreBoot( + deps({ + mode: "redis-read", + hostConfigured: true, + evictionPolicy: async () => ({ + kind: "known", + nodes: [ + { node: "10.0.0.1:6379", policy: "noeviction" }, + { node: "10.0.0.2:6379", policy: "volatile-ttl" }, + ], + }), + }) + ) + ).rejects.toThrow(/10\.0\.0\.2:6379/); + }); + + it("passes when every node reports noeviction", async () => { + const d = deps({ mode: "redis-read", hostConfigured: true }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.policyProbes).toBe(1); + expect(d.recorded.warnings).toHaveLength(0); + }); + + it("warns rather than refusing when the policy cannot be read", async () => { + const d = deps({ + mode: "redis-read", + hostConfigured: true, + evictionPolicy: async () => ({ kind: "unknown", reason: "CONFIG GET is disabled" }), + }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.warnings.join(" ")).toMatch(/maxmemory-policy/i); + }); + + it("warns rather than refusing when no node reported a policy", async () => { + const d = deps({ + mode: "redis-read", + hostConfigured: true, + evictionPolicy: async () => ({ kind: "known", nodes: [] }), + }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.warnings.join(" ")).toMatch(/maxmemory-policy/i); + }); + + it("does not probe the policy at off", async () => { + const d = deps(); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.policyProbes).toBe(0); + }); + + it("does not probe the policy when the endpoint is unreachable", async () => { + const d = deps({ mode: "dual-write", hostConfigured: true, ping: async () => false }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.policyProbes).toBe(0); + }); + + it("warns at redis-only because this build still writes Postgres snapshots", async () => { + const d = deps({ mode: "redis-only", hostConfigured: true }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + expect(d.recorded.warnings.join(" ")).toMatch(/Postgres/i); + }); + + it("validates the configuration whenever a host is set, not only past off", async () => { + // The per-organisation override can put one organisation at dual-write while the deployment dial + // is still off, which is how the ramp starts. Keying these assertions on the deployment dial + // therefore lets a ramped organisation run on a configuration nothing checked. + await expect( + assertSnapshotStoreBoot(deps({ mode: "off", hostConfigured: true, completedTtlMs: 0 })) + ).rejects.toThrow(/COMPLETED_TTL_MS/); + + await expect( + assertSnapshotStoreBoot(deps({ mode: "off", hostConfigured: true, orphanAgeMs: -1 })) + ).rejects.toThrow(/ORPHAN_AGE_MS/); + + await expect( + assertSnapshotStoreBoot(deps({ mode: "off", hostConfigured: true, repairBound: () => false })) + ).rejects.toThrow(/repair/i); + }); + + it("still asks nothing of an unconfigured deployment", async () => { + const d = deps({ mode: "off", hostConfigured: false, completedTtlMs: 0, orphanAgeMs: -1 }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + }); + it("refuses to start when the retired halt environment variable still asks for a halt", async () => { + // Fail loud. The variable no longer does anything, so leaving it set means an operator believes + // the mirror is stopped when it is running. + const d = deps({ legacyEnvHalt: true }); + await expect(assertSnapshotStoreBoot(d)).rejects.toThrow(/snapshotStoreHalt/i); + }); + + it("ignores the retired variable when it is pinned at its old default", async () => { + // "0" carries no intent, and refusing on it would break every deployment that still pins the + // old default while asking for nothing. + const d = deps({ legacyEnvHalt: false }); + await expect(assertSnapshotStoreBoot(d)).resolves.toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/snapshotStoreConfigured.server.test.ts b/apps/webapp/test/snapshotStoreConfigured.server.test.ts new file mode 100644 index 00000000000..666aa4e9e60 --- /dev/null +++ b/apps/webapp/test/snapshotStoreConfigured.server.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { isSnapshotStoreConfigured } from "~/v3/snapshotStoreConfigured.server"; + +describe("isSnapshotStoreConfigured", () => { + it("is false when no Redis host is set (feature merged but inert)", () => { + expect(isSnapshotStoreConfigured(undefined)).toBe(false); + expect(isSnapshotStoreConfigured("")).toBe(false); + }); + + it("is true once a Redis host is set", () => { + expect(isSnapshotStoreConfigured("snap-redis.internal")).toBe(true); + }); +}); diff --git a/apps/webapp/test/snapshotStoreConstruction.test.ts b/apps/webapp/test/snapshotStoreConstruction.test.ts new file mode 100644 index 00000000000..75fc71efd57 --- /dev/null +++ b/apps/webapp/test/snapshotStoreConstruction.test.ts @@ -0,0 +1,86 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// A socket-level assertion is impossible in this suite: test/setup.ts mocks ioredis with a +// LazyRedis subclass that forces lazyConnect, so no client ever dials and a "no connection was +// opened" test would pass even if the gate were broken. The property that does hold, and the one +// that matters, is that no client OBJECT is constructed — an unconstructed client cannot dial. +async function importInstanceModule() { + vi.resetModules(); + delete (globalThis as Record).__trigger_singletons; + return import("~/v3/snapshotStoreInstance.server"); +} + +afterEach(async () => { + vi.unstubAllEnvs(); + vi.resetModules(); + delete (globalThis as Record).__trigger_singletons; +}); + +describe("snapshot store construction gate", () => { + it("constructs nothing when the snapshot-store host is unset", async () => { + // The generic pair is set by test/setup.ts, so this also covers the no-fallback rule: if any + // variable in the block fell back to REDIS_HOST, the store would be constructed here. + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", ""); + + const mod = await importInstanceModule(); + const sentinel = {} as never; + + expect(mod.decorateWithSnapshotStore(sentinel)).toBe(sentinel); + expect(mod.getSnapshotSweepClient()).toBeUndefined(); + expect(mod.getSnapshotStoreConfig().configured).toBe(false); + }); + + it("constructs the decorator and the sweep client once the host is set", async () => { + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", "127.0.0.1"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT", "6379"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED", "true"); + + const mod = await importInstanceModule(); + const sentinel = {} as never; + + // Without this the negative above is satisfiable by an import that throws. + expect(mod.decorateWithSnapshotStore(sentinel)).not.toBe(sentinel); + expect(mod.getSnapshotSweepClient()).toBeDefined(); + expect(mod.getSnapshotStoreConfig().configured).toBe(true); + + await mod.quitSnapshotStoreClients(); + }); + + it("reports the resolved configuration for the boot log line", async () => { + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", "127.0.0.1"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED", "true"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_COMPLETED_TTL_MS", "1000"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_ORPHAN_AGE_MS", "2000"); + + const mod = await importInstanceModule(); + const config = mod.getSnapshotStoreConfig(); + + expect(config).toMatchObject({ + configured: true, + mode: "off", + completedTtlMs: 1000, + orphanAgeMs: 2000, + keyPrefix: "engine:", + clusterMode: false, + }); + + await mod.quitSnapshotStoreClients(); + }); + + it("builds the clients to fail fast instead of queueing while Redis is unreachable", async () => { + // A birth writes Redis before Postgres. With the offline queue on and no command timeout, an + // append issued during a Redis outage waits for a reconnect and the trigger request hangs, so + // run creation stops rather than falling back to Postgres. + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST", "127.0.0.1"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT", "6379"); + vi.stubEnv("RUN_ENGINE_SNAPSHOT_STORE_REDIS_TLS_DISABLED", "true"); + + const mod = await importInstanceModule(); + const config = mod.getSnapshotStoreConfig(); + + expect(config.commandTimeoutMs).toBeGreaterThan(0); + expect(config.offlineQueue).toBe(false); + + await mod.quitSnapshotStoreClients(); + }); +}); diff --git a/apps/webapp/test/snapshotStoreFlagGuard.test.ts b/apps/webapp/test/snapshotStoreFlagGuard.test.ts new file mode 100644 index 00000000000..d7c7edeac7d --- /dev/null +++ b/apps/webapp/test/snapshotStoreFlagGuard.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + globalOnlySnapshotStoreFlagError, + snapshotStoreFlagSaveError, +} from "~/v3/snapshotStoreFlagGuard.server"; +import { + FeatureFlagCatalog, + FEATURE_FLAG, + GLOBAL_LOCKED_FLAGS, + ORG_LOCKED_FLAGS, +} from "~/v3/featureFlags"; + +describe("snapshotStoreFlagSaveError", () => { + it("refuses a flip past off when no host is configured", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "dual-write" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("names the position it refused", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "redis-read" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toMatch(/redis-read/); + }); + + it("allows a flip past off once the host is configured", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "dual-write" }, + { redisHostConfigured: true, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("allows off with no host, because that is the default state", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "off" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("ignores a payload that does not mention the dial", () => { + expect( + snapshotStoreFlagSaveError( + { runOpsMintKind: "cuid" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("ignores a non-string dial value and leaves it to schema validation", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: 3 }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("refuses a per-organisation flip past off when no host is configured", () => { + // Same silence as the global key: without a connection the store is never constructed. + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "dual-write" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("allows a per-organisation off with no host", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "off" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("refuses a per-organisation redis-read when no host is configured", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "redis-read" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("refuses a per-organisation redis-only when no host is configured", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "redis-only" }, + { redisHostConfigured: false, everEnabled: true } + ) + ).toMatch(/RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST/); + }); + + it("allows a per-organisation redis-read once host and latch are set", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "redis-read" }, + { redisHostConfigured: true, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("allows a per-organisation redis-only once host and latch are set", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "redis-only" }, + { redisHostConfigured: true, everEnabled: true } + ) + ).toBeUndefined(); + }); +}); + +describe("globalOnlySnapshotStoreFlagError", () => { + it("refuses the per-organisation key on a global save", () => { + // Nothing reads it from the global row, so a value saved there is an inert control. + expect(globalOnlySnapshotStoreFlagError({ snapshotStoreOrgMode: "dual-write" })).toMatch( + /per-organisation only/ + ); + }); + + it("allows the global key", () => { + expect(globalOnlySnapshotStoreFlagError({ snapshotStoreMode: "dual-write" })).toBeUndefined(); + }); + + it("ignores unrelated payloads", () => { + expect(globalOnlySnapshotStoreFlagError({ runOpsMintKind: "cuid" })).toBeUndefined(); + }); +}); + +describe("the global page and the save guard agree", () => { + it("locks every flag the global save path refuses", () => { + // A flag the guard rejects but the page leaves editable renders a control whose only outcome is + // a 400. The convention above GLOBAL_LOCKED_FLAGS states this; the assertion enforces it. + for (const key of [FEATURE_FLAG.snapshotStoreOrgMode] as const) { + expect(globalOnlySnapshotStoreFlagError({ [key]: "dual-write" })).toBeDefined(); + expect(GLOBAL_LOCKED_FLAGS).toContain(key); + } + }); + + it("leaves the deployment-wide dial editable on the global page", () => { + expect(globalOnlySnapshotStoreFlagError({ snapshotStoreMode: "dual-write" })).toBeUndefined(); + expect(GLOBAL_LOCKED_FLAGS).not.toContain(FEATURE_FLAG.snapshotStoreMode); + }); +}); + +describe("the residency latch", () => { + it("refuses to enable the deployment dial before the latch is set", () => { + // Ordering matters and must be impossible to get wrong. Transitions skip Redis entirely while + // the latch is unset, so a run born after the dial moved but before the latch landed would be + // resident with its transitions skipped, and its head would freeze. Latch first, always. + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "dual-write" }, + { redisHostConfigured: true, everEnabled: false } + ) + ).toMatch(/snapshotStoreEverEnabled/); + }); + + it("refuses to enable a per-organisation override before the latch is set", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreOrgMode: "dual-write" }, + { redisHostConfigured: true, everEnabled: false } + ) + ).toMatch(/snapshotStoreEverEnabled/); + }); + + it("allows enabling once the latch is set", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "dual-write" }, + { redisHostConfigured: true, everEnabled: true } + ) + ).toBeUndefined(); + }); + + it("never blocks a move back to off, whatever the latch says", () => { + // Turning it down must never be gated. That is the rollback path. + expect( + snapshotStoreFlagSaveError( + { snapshotStoreMode: "off" }, + { redisHostConfigured: true, everEnabled: false } + ) + ).toBeUndefined(); + }); + + it("does not block setting the latch itself", () => { + expect( + snapshotStoreFlagSaveError( + { snapshotStoreEverEnabled: true }, + { redisHostConfigured: true, everEnabled: false } + ) + ).toBeUndefined(); + }); + + it("is a deployment-wide flag, and takes only a real boolean", () => { + expect(FeatureFlagCatalog.snapshotStoreEverEnabled.safeParse(true).success).toBe(true); + expect(FeatureFlagCatalog.snapshotStoreEverEnabled.safeParse("true").success).toBe(false); + expect(ORG_LOCKED_FLAGS).toContain("snapshotStoreEverEnabled"); + }); +}); diff --git a/apps/webapp/test/snapshotStoreFlags.test.ts b/apps/webapp/test/snapshotStoreFlags.test.ts new file mode 100644 index 00000000000..453f31a1e8a --- /dev/null +++ b/apps/webapp/test/snapshotStoreFlags.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { + FeatureFlagCatalog, + ORG_LOCKED_FLAGS, + withoutOrgForbiddenSnapshotKeys, +} from "~/v3/featureFlags"; + +describe("snapshot store dial catalog", () => { + it("accepts all four positions globally", () => { + for (const value of ["off", "dual-write", "redis-read", "redis-only"]) { + expect(FeatureFlagCatalog.snapshotStoreMode.safeParse(value).success).toBe(true); + } + }); + + it("rejects an unknown position", () => { + expect(FeatureFlagCatalog.snapshotStoreMode.safeParse("redis-write").success).toBe(false); + }); + + it("accepts all four positions per organisation, so an org can be soaked at a read position", () => { + for (const value of ["off", "dual-write", "redis-read", "redis-only"]) { + expect(FeatureFlagCatalog.snapshotStoreOrgMode.safeParse(value).success).toBe(true); + } + expect(FeatureFlagCatalog.snapshotStoreOrgMode.safeParse("redis-write").success).toBe(false); + }); + + it("lists the global dial as org-locked", () => { + expect(ORG_LOCKED_FLAGS).toContain("snapshotStoreMode"); + }); +}); + +describe("withoutOrgForbiddenSnapshotKeys", () => { + it("removes the global dial and keeps everything else", () => { + expect( + withoutOrgForbiddenSnapshotKeys({ + snapshotStoreMode: "redis-only", + snapshotStoreOrgMode: "dual-write", + runOpsMintKind: "cuid", + }) + ).toEqual({ snapshotStoreOrgMode: "dual-write", runOpsMintKind: "cuid" }); + }); + + it("returns the same object when the dial is absent", () => { + const input = { runOpsMintKind: "cuid" }; + expect(withoutOrgForbiddenSnapshotKeys(input)).toBe(input); + }); + + it("removes the dial even when it is the only key", () => { + expect(withoutOrgForbiddenSnapshotKeys({ snapshotStoreMode: "dual-write" })).toEqual({}); + }); +}); diff --git a/apps/webapp/test/snapshotStoreGlobalModeEverEnabled.test.ts b/apps/webapp/test/snapshotStoreGlobalModeEverEnabled.test.ts new file mode 100644 index 00000000000..806e27a7803 --- /dev/null +++ b/apps/webapp/test/snapshotStoreGlobalModeEverEnabled.test.ts @@ -0,0 +1,152 @@ +// The global one-way "mode ever non-off" latch. It records whether the global dial has ever been +// past `off`, so a per-org transition-skip can tell a never-enabled org from one that saw a +// global-era birth. One-way: set on the first non-off save, never cleared, never deleted by the +// replace-semantics sweep. NEVER mocks the DB: real testcontainers Postgres FeatureFlag rows. +import type { PrismaClient } from "@trigger.dev/database"; +import { postgresTest } from "@internal/testcontainers"; +import { describe, expect, it, vi } from "vitest"; +import { + FEATURE_FLAG, + FeatureFlagCatalog, + stampSnapshotStoreGlobalModeEverEnabled, + withoutOrgForbiddenSnapshotKeys, + type FeatureFlagKey, +} from "~/v3/featureFlags"; +import { + makeSetMultipleFlags, + replaceGlobalFeatureFlags, + setGlobalFeatureFlagsTransactional, + stampGlobalModeLatchForMerge, +} from "~/v3/featureFlags.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +const MODE = FEATURE_FLAG.snapshotStoreMode; +const LATCH = FEATURE_FLAG.snapshotStoreGlobalModeEverEnabled; +const CATALOG_KEYS = Object.keys(FeatureFlagCatalog) as FeatureFlagKey[]; + +async function readFlag(prisma: PrismaClient, key: FeatureFlagKey): Promise { + const row = await prisma.featureFlag.findFirst({ where: { key }, select: { value: true } }); + return row?.value; +} + +function replace( + prisma: PrismaClient, + requestedFlags: Record, + opts?: { unlockLockedFlags?: boolean; isManagedCloud?: boolean } +) { + return replaceGlobalFeatureFlags(prisma, { + requestedFlags, + catalogKeys: CATALOG_KEYS, + isManagedCloud: opts?.isManagedCloud ?? false, + unlockLockedFlags: opts?.unlockLockedFlags ?? false, + graceMs: 0, + }); +} + +describe("stampSnapshotStoreGlobalModeEverEnabled (pure one-way latch)", () => { + it("latches true when the resulting dial moves past off", () => { + for (const mode of ["dual-write", "redis-read", "redis-only"]) { + const stamped = stampSnapshotStoreGlobalModeEverEnabled(null, { [MODE]: mode }); + expect(stamped[LATCH], mode).toBe(true); + } + }); + + it("keeps the latch true when the dial is set back to off (one-way)", () => { + const stamped = stampSnapshotStoreGlobalModeEverEnabled({ [LATCH]: true }, { [MODE]: "off" }); + expect(stamped[LATCH]).toBe(true); + }); + + it("carries an existing latch forward on a save that omits the dial", () => { + const stamped = stampSnapshotStoreGlobalModeEverEnabled({ [LATCH]: true }, { someOther: "x" }); + expect(stamped[LATCH]).toBe(true); + }); + + it("leaves the latch absent (never false) when off and never enabled", () => { + const stamped = stampSnapshotStoreGlobalModeEverEnabled(null, { [MODE]: "off" }); + expect(LATCH in stamped).toBe(false); + }); + + it("is stripped from an operator-supplied org save payload", () => { + expect( + withoutOrgForbiddenSnapshotKeys({ [LATCH]: false, [FEATURE_FLAG.mollifierEnabled]: true }) + ).toEqual({ [FEATURE_FLAG.mollifierEnabled]: true }); + }); +}); + +describe("replaceGlobalFeatureFlags — global mode latch (replace semantics)", () => { + postgresTest("stamps the latch true when the dial is saved to dual-write", async ({ prisma }) => { + await replace(prisma, { [MODE]: "dual-write" }); + expect(await readFlag(prisma, LATCH)).toBe(true); + }); + + postgresTest("redis-read and redis-only also stamp the latch", async ({ prisma }) => { + for (const mode of ["redis-read", "redis-only"]) { + await replace(prisma, { [MODE]: mode }); + expect(await readFlag(prisma, LATCH), mode).toBe(true); + await prisma.featureFlag.deleteMany({ where: { key: { in: [LATCH, MODE] } } }); + } + }); + + postgresTest( + "keeps the latch when a later save sets the dial back to off", + async ({ prisma }) => { + await replace(prisma, { [MODE]: "dual-write" }); + await replace(prisma, { [MODE]: "off" }); + expect(await readFlag(prisma, MODE)).toBe("off"); + expect(await readFlag(prisma, LATCH)).toBe(true); + } + ); + + postgresTest("does not stamp the latch when off and never enabled", async ({ prisma }) => { + await replace(prisma, { [MODE]: "off" }); + expect(await readFlag(prisma, LATCH)).toBeUndefined(); + }); + + postgresTest("keeps the latch even on a self-hosted unlock that omits it", async ({ prisma }) => { + await replace(prisma, { [MODE]: "dual-write" }); + // Unlock + omit the latch: without protection the replace-sweep would delete it. + await replace(prisma, {}, { unlockLockedFlags: true }); + expect(await readFlag(prisma, LATCH)).toBe(true); + }); + + postgresTest("never writes a false latch from an operator-supplied value", async ({ prisma }) => { + // A self-hosted unlock is the only way the latch key reaches the payload; it must not win. + await replace(prisma, { [LATCH]: false, [MODE]: "off" }, { unlockLockedFlags: true }); + expect(await readFlag(prisma, LATCH)).toBeUndefined(); + }); +}); + +describe("stampGlobalModeLatchForMerge — JSON admin API (merge semantics)", () => { + postgresTest("stamps the latch true when enabling via merge", async ({ prisma }) => { + const stamped = await stampGlobalModeLatchForMerge(prisma, { [MODE]: "dual-write" }); + expect(stamped[LATCH]).toBe(true); + }); + + postgresTest("carries a stored latch forward on a save back to off", async ({ prisma }) => { + await makeSetMultipleFlags(prisma)({ [LATCH]: true }); + const stamped = await stampGlobalModeLatchForMerge(prisma, { [MODE]: "off" }); + expect(stamped[LATCH]).toBe(true); + }); + + postgresTest("does not stamp when off and no stored latch", async ({ prisma }) => { + const stamped = await stampGlobalModeLatchForMerge(prisma, { [MODE]: "off" }); + expect(LATCH in stamped).toBe(false); + }); + + postgresTest("orders the latch before the mode for a crash-safe write", async ({ prisma }) => { + // makeSetMultipleFlags upserts in insertion order, so the latch must come first: a crash mid-write + // then leaves latch=true with the mode possibly still off, never mode=non-off + latch absent. + const stamped = await stampGlobalModeLatchForMerge(prisma, { [MODE]: "dual-write" }); + const keys = Object.keys(stamped); + expect(keys.indexOf(LATCH)).toBe(0); + expect(keys.indexOf(LATCH)).toBeLessThan(keys.indexOf(MODE)); + }); + + postgresTest("transactional write lands both the mode and the latch", async ({ prisma }) => { + const stamped = await stampGlobalModeLatchForMerge(prisma, { [MODE]: "dual-write" }); + await setGlobalFeatureFlagsTransactional(prisma, stamped); + expect(await readFlag(prisma, MODE)).toBe("dual-write"); + expect(await readFlag(prisma, LATCH)).toBe(true); + }); +}); diff --git a/apps/webapp/test/snapshotStoreHalt.test.ts b/apps/webapp/test/snapshotStoreHalt.test.ts new file mode 100644 index 00000000000..bd1dbc076c7 --- /dev/null +++ b/apps/webapp/test/snapshotStoreHalt.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { + FeatureFlagCatalog, + ORG_LOCKED_FLAGS, + withoutOrgForbiddenSnapshotKeys, +} from "~/v3/featureFlags"; +import { buildSnapshotStoreHaltCheck } from "~/v3/snapshotStoreMode.server"; + +describe("the snapshot store halt check", () => { + it("is not halted by default", () => { + expect(buildSnapshotStoreHaltCheck({ flag: () => undefined })()).toBe(false); + expect(buildSnapshotStoreHaltCheck({ flag: () => false })()).toBe(false); + }); + + it("halts on the flag, so an incident needs no deploy", () => { + expect(buildSnapshotStoreHaltCheck({ flag: () => true })()).toBe(true); + }); + + it("is the flag and nothing else, so it converges in one flag interval", () => { + // The environment half is gone on purpose. It converged over a rolling deploy instead of a + // flag interval, and during that window a halted process skips a transition while an unhalted + // one asserts a head it cannot see, which forks once per process flip per run. A control whose + // own convergence manufactures the divergence it exists to stop is not a control. + expect(buildSnapshotStoreHaltCheck({ flag: () => false })()).toBe(false); + expect(buildSnapshotStoreHaltCheck({ flag: () => true })()).toBe(true); + }); +}); + +describe("the halt flag", () => { + it("takes only a real boolean, so a stringified value cannot enable or disable it", () => { + expect(FeatureFlagCatalog.snapshotStoreHalt.safeParse(true).success).toBe(true); + expect(FeatureFlagCatalog.snapshotStoreHalt.safeParse(false).success).toBe(true); + expect(FeatureFlagCatalog.snapshotStoreHalt.safeParse("true").success).toBe(false); + }); + + it("is deployment-wide only", () => { + expect(ORG_LOCKED_FLAGS).toContain("snapshotStoreHalt"); + expect( + withoutOrgForbiddenSnapshotKeys({ snapshotStoreHalt: true, runOpsMintKind: "cuid" }) + ).toEqual({ runOpsMintKind: "cuid" }); + }); +}); diff --git a/apps/webapp/test/snapshotStoreMetrics.test.ts b/apps/webapp/test/snapshotStoreMetrics.test.ts new file mode 100644 index 00000000000..8a806a81f98 --- /dev/null +++ b/apps/webapp/test/snapshotStoreMetrics.test.ts @@ -0,0 +1,76 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { APPEND_RESULT_OUTCOMES } from "@internal/run-store"; +import { WRITE_OUTCOMES } from "~/v3/snapshotStoreMetrics.server"; + +const SOURCE_PATH = join(process.cwd(), "app/v3/snapshotStoreMetrics.server.ts"); + +describe("snapshotStoreMetrics module shape", () => { + // An instrument created but never incremented emits no data point, so a MeterProvider cannot see + // one that drifted back to module scope, where it would register on every boot. + const source = readFileSync(SOURCE_PATH, "utf8"); + const factoryAt = source.indexOf("export function createSnapshotStoreMetrics"); + + it("exports the factory", () => { + expect(factoryAt).toBeGreaterThan(-1); + }); + + it("creates every instrument inside the factory", () => { + const creations = [ + ...source.matchAll(/create(Counter|Histogram|UpDownCounter|Observable\w*)\(/g), + ]; + expect(creations.length).toBeGreaterThan(0); + for (const match of creations) { + expect(match.index).toBeGreaterThan(factoryAt); + } + }); + + it("calls getMeter nowhere at module scope", () => { + expect(source).not.toMatch(/^\s*(const|let|var)\s+\w+\s*=\s*getMeter\(/m); + }); + + it("declares no counter that has no producer in this ticket", () => { + // A counter pinned at zero looks the same as a working one that found nothing. + expect(source).not.toMatch(/compare_divergence/); + expect(source).not.toMatch(/\btrimmed\b/); + }); + + it("bounds the write outcome against the store's own vocabulary", () => { + // Any outcome the store can return but the allowlist omits collapses to "other", which hides + // forked appends: the signal that a run's Redis head has frozen. + for (const outcome of APPEND_RESULT_OUTCOMES) { + expect(WRITE_OUTCOMES).toContain(outcome); + } + }); + + it("never names an attribute `source`", () => { + // Every exported series already carries a `source` label naming the telemetry pipeline. A data + // point that repeats the name is dropped, so the metric vanishes while its counter is still + // being incremented. This cost an hour of chasing a phantom read path. + const recorders = source.slice(source.indexOf("const decorator")); + expect(recorders).not.toMatch(/^\s*source:/m); + expect(recorders).toMatch(/served_by:/); + }); + + it("declares no counter whose only producer is unreachable", () => { + // recordWrite is called once, with an AppendResult outcome. "staged" and "post_expiry" are not + // in that vocabulary, so both counters sat at zero and both branches were dead. + expect(source).not.toMatch(/flush_staged/); + expect(source).not.toMatch(/post_expiry_write/); + }); + + it("gives the two layers separate counters", () => { + // Sharing one would count a single logical write twice and mix {outcome, ttl} points with + // {site, outcome} points under one name. + const appendBlock = source.slice( + source.indexOf("recordAppend:"), + source.indexOf("recordWrite:") + ); + const writeBlock = source.slice(source.indexOf("recordWrite:")); + expect(appendBlock).toMatch(/appendTotal\.add/); + expect(appendBlock).not.toMatch(/writeTotal\.add/); + expect(writeBlock).toMatch(/writeTotal\.add/); + expect(writeBlock).not.toMatch(/appendTotal\.add/); + }); +}); diff --git a/apps/webapp/test/snapshotStoreMetricsEmit.test.ts b/apps/webapp/test/snapshotStoreMetricsEmit.test.ts new file mode 100644 index 00000000000..b8bb58ec8c1 --- /dev/null +++ b/apps/webapp/test/snapshotStoreMetricsEmit.test.ts @@ -0,0 +1,81 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { createSnapshotStoreMetrics } from "~/v3/snapshotStoreMetrics.server"; +import { createInMemoryMetrics } from "./utils/tracing"; +import { latestMetrics, metricSum } from "./otlpMetrics.helpers"; + +const COHORT = "org_soak"; + +describe("snapshot store metrics per-org label", () => { + let helper: ReturnType | undefined; + + afterEach(async () => { + await helper?.shutdown(); + helper = undefined; + }); + + it("labels a cohort org's append with its own id and a non-member with 'other'", async () => { + helper = createInMemoryMetrics(); + const { store } = createSnapshotStoreMetrics(helper.meter, (orgId) => orgId === COHORT); + + store.recordAppend("written", "none", COHORT); + store.recordAppend("written", "none", "org_other"); + store.recordAppend("written", "none", undefined); + + const metrics = await latestMetrics(helper); + expect( + metricSum(metrics, "run_engine.snapshot_store.append_total", { + outcome: "written", + org: COHORT, + }) + ).toBe(1); + expect( + metricSum(metrics, "run_engine.snapshot_store.append_total", { + outcome: "written", + org: "other", + }) + ).toBe(2); + }); + + it("labels appendFailed with the cohort org id, else 'other'", async () => { + helper = createInMemoryMetrics(); + const { decorator } = createSnapshotStoreMetrics(helper.meter, (orgId) => orgId === COHORT); + + decorator.recordAppendFailed("createRun", COHORT); + decorator.recordAppendFailed("createRun", "org_other"); + + const metrics = await latestMetrics(helper); + expect( + metricSum(metrics, "run_engine.snapshot_store.append_failed", { + site: "createRun", + org: COHORT, + }) + ).toBe(1); + expect( + metricSum(metrics, "run_engine.snapshot_store.append_failed", { + site: "createRun", + org: "other", + }) + ).toBe(1); + }); + + it("defaults every org to 'other' when no cohort predicate is supplied", async () => { + helper = createInMemoryMetrics(); + const { store } = createSnapshotStoreMetrics(helper.meter); + + store.recordAppend("written", "none", COHORT); + + const metrics = await latestMetrics(helper); + expect( + metricSum(metrics, "run_engine.snapshot_store.append_total", { + outcome: "written", + org: COHORT, + }) + ).toBe(0); + expect( + metricSum(metrics, "run_engine.snapshot_store.append_total", { + outcome: "written", + org: "other", + }) + ).toBe(1); + }); +}); diff --git a/apps/webapp/test/snapshotStoreMode.test.ts b/apps/webapp/test/snapshotStoreMode.test.ts new file mode 100644 index 00000000000..c381e58359a --- /dev/null +++ b/apps/webapp/test/snapshotStoreMode.test.ts @@ -0,0 +1,459 @@ +import { describe, expect, it, vi } from "vitest"; +import type { SnapshotStoreMode } from "@internal/run-store"; +import { + createOrgModeSource, + buildSnapshotStoreModeResolver, + cachedOrgModeFor, + NO_OVERRIDE, +} from "~/v3/snapshotStoreMode.server"; + +function build(opts: { + globalMode?: SnapshotStoreMode; + perOrg?: Record; + envFloor?: SnapshotStoreMode; + refresh?: (organizationId: string) => void; +}) { + return buildSnapshotStoreModeResolver({ + globalMode: () => opts.globalMode, + orgMode: { + get: (id: string) => opts.perOrg?.[id], + refresh: opts.refresh ?? (() => {}), + }, + envFloor: opts.envFloor ?? "off", + }); +} + +describe("snapshot store mode resolver", () => { + it("falls back to the env floor when the global snapshot is cold", () => { + expect(build({ envFloor: "off" }).resolve()).toBe("off"); + expect(build({ envFloor: "dual-write" }).resolve()).toBe("dual-write"); + }); + + it("prefers the global flag over the floor", () => { + expect(build({ globalMode: "redis-read", envFloor: "off" }).resolve()).toBe("redis-read"); + }); + + it("prefers an organisation override over the global flag", () => { + const r = build({ globalMode: "off", perOrg: { org_a: "dual-write" } }); + expect(r.resolve("org_a")).toBe("dual-write"); + expect(r.resolve("org_b")).toBe("off"); + }); + + it("lets an organisation be off while the global flag is on", () => { + const r = build({ globalMode: "dual-write", perOrg: { org_a: "off" } }); + expect(r.resolve("org_a")).toBe("off"); + expect(r.resolve("org_b")).toBe("dual-write"); + }); + + it("serves the global answer on a cold organisation and schedules a refresh", () => { + const refresh = vi.fn(); + const r = build({ globalMode: "dual-write", refresh }); + expect(r.resolve("org_cold")).toBe("dual-write"); + expect(refresh).toHaveBeenCalledWith("org_cold"); + }); + + it("never lets a refresh failure reach the caller", () => { + const refresh = vi.fn(() => { + throw new Error("control plane unreachable"); + }); + const r = build({ globalMode: "off", refresh }); + expect(() => r.resolve("org_x")).not.toThrow(); + expect(r.resolve("org_x")).toBe("off"); + }); + + it("resolves an unknown organisation to the global answer, never a throw", () => { + const r = build({ globalMode: "off", perOrg: {} }); + expect(r.resolve("org_deleted")).toBe("off"); + }); + + it("caches an absent override rather than nothing", () => { + // Caching nothing means every organisation without an override re-queries on every write. + expect(cachedOrgModeFor(undefined)).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor(null)).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor("not-a-mode")).toBe(NO_OVERRIDE); + expect(cachedOrgModeFor("dual-write")).toBe("dual-write"); + expect(cachedOrgModeFor("redis-read")).toBe("redis-read"); + expect(cachedOrgModeFor("redis-only")).toBe("redis-only"); + }); + + it("stops querying once an absent override is cached", () => { + // Without a cached negative, every organisation with no override re-queries on every write, + // which is every organisation until a ramp starts. + const refresh = vi.fn(); + let cached: string | undefined; + const r = buildSnapshotStoreModeResolver({ + globalMode: () => "off", + orgMode: { + get: () => cached as never, + refresh: (id: string) => { + refresh(id); + cached = "__none__"; + }, + }, + envFloor: "off", + }); + + expect(r.resolve("org_a")).toBe("off"); + expect(refresh).toHaveBeenCalledTimes(1); + + expect(r.resolve("org_a")).toBe("off"); + expect(r.resolve("org_a")).toBe("off"); + expect(refresh).toHaveBeenCalledTimes(1); + }); + + it("does not consult the organisation source when no organisation is supplied", () => { + const get = vi.fn(() => undefined); + const r = buildSnapshotStoreModeResolver({ + globalMode: () => "redis-read", + orgMode: { get, refresh: () => {} }, + envFloor: "off", + }); + expect(r.resolve()).toBe("redis-read"); + expect(get).not.toHaveBeenCalled(); + }); +}); + +describe("the org-scoped read routing", () => { + function buildRead(opts: { + globalMode?: SnapshotStoreMode; + perOrg?: Record; + runToOrg?: Record; + census?: { anyOrgReadEnabled: boolean; anyOrgRedisOnly: boolean }; + }) { + return buildSnapshotStoreModeResolver({ + globalMode: () => opts.globalMode, + orgMode: { + get: (id: string) => opts.perOrg?.[id], + refresh: () => {}, + }, + runOrg: { resolve: (runId: string) => opts.runToOrg?.[runId] }, + census: opts.census + ? { + anyOrgReadEnabled: () => opts.census!.anyOrgReadEnabled, + anyOrgRedisOnly: () => opts.census!.anyOrgRedisOnly, + } + : undefined, + envFloor: "off", + }); + } + + it("routes a run in a redis-read org to that org's read position", () => { + const r = buildRead({ + globalMode: "off", + perOrg: { org_a: "redis-read" }, + runToOrg: { run_1: "org_a" }, + }); + expect(r.readModeFor?.("run_1")).toBe("redis-read"); + }); + + it("returns undefined for a run whose org cannot be resolved, so the decorator falls back", () => { + const r = buildRead({ globalMode: "redis-read", runToOrg: {} }); + expect(r.readModeFor?.("run_unknown")).toBeUndefined(); + }); + + it("returns the global answer for a resolved run whose org has no override", () => { + const r = buildRead({ globalMode: "dual-write", runToOrg: { run_1: "org_a" }, perOrg: {} }); + expect(r.readModeFor?.("run_1")).toBe("dual-write"); + }); + + it("delegates the cheap read gates to the census", () => { + const r = buildRead({ + globalMode: "off", + census: { anyOrgReadEnabled: true, anyOrgRedisOnly: false }, + }); + expect(r.anyOrgReadEnabled?.()).toBe(true); + expect(r.anyOrgRedisOnly?.()).toBe(false); + }); + + it("is inert when no run→org source or census is wired", () => { + const r = buildSnapshotStoreModeResolver({ + globalMode: () => "off", + orgMode: { get: () => undefined, refresh: () => {} }, + envFloor: "off", + }); + expect(r.readModeFor?.("run_1")).toBeUndefined(); + expect(r.anyOrgReadEnabled?.()).toBe(false); + expect(r.anyOrgRedisOnly?.()).toBe(false); + }); +}); + +describe("the authoritative read position (redis-only fallback gate)", () => { + function buildAuth(opts: { + globalMode?: SnapshotStoreMode; + perOrg?: Record; + resolveAuthoritative?: (runId: string) => Promise; + warm?: (organizationId: string) => Promise; + }) { + return buildSnapshotStoreModeResolver({ + globalMode: () => opts.globalMode, + orgMode: { + get: (id: string) => opts.perOrg?.[id], + refresh: () => {}, + ...(opts.warm && { warm: opts.warm }), + }, + runOrg: { + resolve: () => undefined, + ...(opts.resolveAuthoritative && { resolveAuthoritative: opts.resolveAuthoritative }), + }, + envFloor: "off", + }); + } + + it("resolves the run's org authoritatively, warms the dial, and returns the org mode", async () => { + const warmed: string[] = []; + const r = buildAuth({ + globalMode: "redis-read", + perOrg: { org_ro: "redis-only" }, + resolveAuthoritative: async () => "org_ro", + warm: async (id) => { + warmed.push(id); + }, + }); + await expect(r.readModeForAuthoritative?.("run_1")).resolves.toBe("redis-only"); + expect(warmed).toContain("org_ro"); + }); + + it("returns a non-redis-only mode for a pre-cutover run so the decorator falls back", async () => { + const r = buildAuth({ + globalMode: "redis-read", + perOrg: {}, + resolveAuthoritative: async () => "org_pre", + }); + await expect(r.readModeForAuthoritative?.("run_1")).resolves.toBe("redis-read"); + }); + + it("propagates a throw from the authoritative run→org read so the decorator fails closed", async () => { + const r = buildAuth({ + globalMode: "redis-read", + resolveAuthoritative: async () => { + throw new Error("run→org read timed out"); + }, + }); + await expect(r.readModeForAuthoritative?.("run_1")).rejects.toThrow(/timed out/); + }); + + it("returns undefined when no authoritative run→org source is wired", async () => { + const r = buildAuth({ globalMode: "redis-read" }); + await expect(r.readModeForAuthoritative?.("run_1")).resolves.toBeUndefined(); + }); +}); + +describe("a saved organisation dial survives a lagging replica", () => { + type Deferred = { resolve: (v: unknown) => void; promise: Promise }; + + function deferred(): Deferred { + let resolve!: (v: unknown) => void; + const promise = new Promise((r) => (resolve = r)); + return { resolve, promise }; + } + + function clientFor(read: () => Promise) { + return { organization: { findFirst: () => read() as never } } as never; + } + + it("does not let a replica read that starts after the save re-cache the old value", async () => { + // The primary carries the saved value; the replica is still lagging and carries the old one. + const primary = deferred(); + const replica = deferred(); + + const source = createOrgModeSource({ + primary: clientFor(() => primary.promise), + replica: clientFor(() => replica.promise), + }); + + // The save path invalidates, which drops the cache and reads the primary. + source.invalidate("org_1"); + // A concurrent write for the same organisation misses the now-empty cache and warms off-path. + source.refresh("org_1"); + + // The primary lands first with the saved value. + primary.resolve({ featureFlags: { snapshotStoreOrgMode: "dual-write" } }); + await new Promise((r) => setTimeout(r, 0)); + expect(source.get("org_1")).toBe("dual-write"); + + // Then the lagging replica lands with the pre-save value. It must not win. + replica.resolve({ featureFlags: {} }); + await new Promise((r) => setTimeout(r, 0)); + + expect(source.get("org_1")).toBe("dual-write"); + }); + it("keeps the save protected when two invalidations for one organisation overlap", async () => { + // primaryPending was a Set, so the FIRST primary read's finally cleared it while the SECOND was + // still in flight. A refresh arriving in that window then started a replica read carrying the + // current generation, so the generation guard could not discard it, and a lagging replica put + // the pre-save value back for a full cache TTL. + const firstPrimary = deferred(); + const secondPrimary = deferred(); + const replica = deferred(); + const primaries = [firstPrimary, secondPrimary]; + let primaryCalls = 0; + + const source = createOrgModeSource({ + primary: clientFor(() => primaries[primaryCalls++]!.promise), + replica: clientFor(() => replica.promise), + }); + + // Two saves for the same organisation, overlapping. + source.invalidate("org_1"); + source.invalidate("org_1"); + + // The FIRST primary read completes. The second is still outstanding, so the organisation must + // still count as pending. + firstPrimary.resolve({ featureFlags: { snapshotStoreOrgMode: "off" } }); + await new Promise((r) => setTimeout(r, 0)); + + // A concurrent read arrives while the second save is still reading. It must not start a replica + // read, because the authoritative answer is still on its way. + source.refresh("org_1"); + + // The second save lands with the value that must win. + secondPrimary.resolve({ featureFlags: { snapshotStoreOrgMode: "dual-write" } }); + await new Promise((r) => setTimeout(r, 0)); + expect(source.get("org_1")).toBe("dual-write"); + + // NOW the lagging replica lands, carrying the pre-save value. It shares the second save's + // generation, so the generation guard cannot discard it: only never having started can stop it. + replica.resolve({ featureFlags: {} }); + await new Promise((r) => setTimeout(r, 0)); + + expect(source.get("org_1")).toBe("dual-write"); + }); +}); + +describe("warming the organisation dial before a birth", () => { + type Deferred = { resolve: (v: unknown) => void; promise: Promise }; + function deferred(): Deferred { + let resolve!: (v: unknown) => void; + const promise = new Promise((r) => (resolve = r)); + return { resolve, promise }; + } + function clientFor(read: () => Promise) { + return { organization: { findFirst: () => read() as never } } as never; + } + + it("resolves once the organisation's value is cached, so a birth sees the truth", async () => { + const replica = deferred(); + const source = createOrgModeSource({ + primary: clientFor(() => Promise.resolve({})), + replica: clientFor(() => replica.promise), + }); + + expect(source.get("org_1")).toBeUndefined(); + + const warming = source.warm("org_1"); + replica.resolve({ featureFlags: { snapshotStoreOrgMode: "dual-write" } }); + await warming; + + // The point of the whole exercise: after warm, the cache holds the real value, so the + // synchronous resolve a birth then performs no longer falls back to the global position. + expect(source.get("org_1")).toBe("dual-write"); + }); + + it("costs nothing when the value is already cached", async () => { + let reads = 0; + const source = createOrgModeSource({ + primary: clientFor(() => Promise.resolve({})), + replica: clientFor(() => { + reads += 1; + return Promise.resolve({ featureFlags: { snapshotStoreOrgMode: "off" } }); + }), + }); + + await source.warm("org_1"); + expect(reads).toBe(1); + expect(source.get("org_1")).toBe("off"); + + // A warm organisation is the common case once it has any traffic, and must not re-read. + await source.warm("org_1"); + expect(reads).toBe(1); + }); + + it("gives up rather than holding a birth open on a slow read", async () => { + // A birth is on the trigger path and the caller may already hold an open transaction, so this + // must be bounded. Giving up restores the previous behaviour, it does not fail the trigger. + const neverResolves = new Promise(() => {}); + const source = createOrgModeSource({ + primary: clientFor(() => Promise.resolve({})), + replica: clientFor(() => neverResolves), + }); + + const started = Date.now(); + await expect(source.warm("org_1")).resolves.toBeUndefined(); + const elapsed = Date.now() - started; + + // Bounded, and nowhere near indefinite. + expect(elapsed).toBeLessThan(2_000); + // Still unknown, so the synchronous resolve falls back exactly as it did before. + expect(source.get("org_1")).toBeUndefined(); + }); + it("keeps replica refreshes out while a failed primary read is still retrying", async () => { + // While the primary is failing and retries are pending, a lagging replica must not restore the + // pre-save value: primaryPending stays set for the generation, so refresh does not read the replica. + let primaryCalls = 0; + const source = createOrgModeSource( + { + primary: clientFor(() => { + primaryCalls += 1; + return Promise.reject(new Error("primary unavailable")); + }), + replica: clientFor(() => + Promise.resolve({ featureFlags: { snapshotStoreOrgMode: "off" } }) + ), + }, + // Long delays: the retry window is still open for the duration of this test. + { primaryInvalidateRetryDelaysMs: [10_000, 10_000] } + ); + + source.invalidate("org_1"); + await vi.waitFor(() => expect(primaryCalls).toBeGreaterThanOrEqual(1)); + + // A refresh arriving mid-retry must not start a replica read; the resolver falls back to global. + source.refresh("org_1"); + await new Promise((r) => setTimeout(r, 0)); + expect(source.get("org_1")).toBeUndefined(); + }); + + it("retries the primary and recovers the saved value after a transient failure", async () => { + let calls = 0; + const source = createOrgModeSource( + { + primary: clientFor(() => { + calls += 1; + return calls === 1 + ? Promise.reject(new Error("primary unavailable")) + : Promise.resolve({ featureFlags: { snapshotStoreOrgMode: "dual-write" } }); + }), + replica: clientFor(() => Promise.resolve({ featureFlags: {} })), + }, + { primaryInvalidateRetryDelaysMs: [5, 5, 5] } + ); + + source.invalidate("org_1"); + // A concurrent refresh during the retry must not read the replica out from under the retry. + source.refresh("org_1"); + + await vi.waitFor(() => expect(source.get("org_1")).toBe("dual-write")); + expect(calls).toBeGreaterThanOrEqual(2); + }); + + it("unwedges after exhausting primary retries so a later refresh repopulates", async () => { + const source = createOrgModeSource( + { + primary: clientFor(() => Promise.reject(new Error("primary unavailable"))), + replica: clientFor(() => + Promise.resolve({ featureFlags: { snapshotStoreOrgMode: "redis-read" } }) + ), + }, + { primaryInvalidateRetryDelaysMs: [1, 1] } + ); + + source.invalidate("org_1"); + + // Once the retries exhaust, primaryPending clears and a refresh can repopulate from the replica, + // rather than the org staying wedged on the global position forever. + await vi.waitFor(() => { + source.refresh("org_1"); + expect(source.get("org_1")).toBe("redis-read"); + }); + }); +}); diff --git a/apps/webapp/test/snapshotStoreModuleLoad.test.ts b/apps/webapp/test/snapshotStoreModuleLoad.test.ts new file mode 100644 index 00000000000..50ed8da596b --- /dev/null +++ b/apps/webapp/test/snapshotStoreModuleLoad.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from "vitest"; + +// Reproduces the CI break. Many webapp suites mock ~/db.server and ~/env.server with minimal +// objects, so a module-load side effect that reads a new env variable takes the whole file down on +// import. This is the exact mock shape of the suite that caught it. +vi.mock("~/db.server", () => ({ prisma: {}, $replica: {} })); +vi.mock("~/env.server", () => ({ env: { SESSION_SECRET: "test-session-secret" } })); +vi.mock("~/services/logger.server", () => ({ + logger: { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} }, +})); + +describe("snapshot store modules under a minimal env mock", () => { + it("imports the mode resolver without constructing anything", async () => { + await expect(import("~/v3/snapshotStoreMode.server")).resolves.toBeDefined(); + }); + + it("resolves off rather than throwing", async () => { + const { snapshotStoreModeResolver } = await import("~/v3/snapshotStoreMode.server"); + expect(snapshotStoreModeResolver.resolve()).toBe("off"); + expect(snapshotStoreModeResolver.resolve("org_anything")).toBe("off"); + }); + + it("imports the instance module and stays undecorated", async () => { + const mod = await import("~/v3/snapshotStoreInstance.server"); + const sentinel = {} as never; + expect(mod.decorateWithSnapshotStore(sentinel)).toBe(sentinel); + expect(mod.getSnapshotSweepClient()).toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/snapshotStoreOrgCensus.server.test.ts b/apps/webapp/test/snapshotStoreOrgCensus.server.test.ts new file mode 100644 index 00000000000..7c4449ae53e --- /dev/null +++ b/apps/webapp/test/snapshotStoreOrgCensus.server.test.ts @@ -0,0 +1,224 @@ +import { describe, expect, it } from "vitest"; +import { + createSnapshotStoreOrgCensus, + defaultCensusAutoStart, + type SnapshotStoreOrgCensusClient, +} from "~/v3/snapshotStoreOrgCensus.server"; + +type Row = { id: string; featureFlags: unknown }; + +type FindManyArgs = Parameters[0]; + +function fakeClient(rows: Row[], calls?: FindManyArgs[]): SnapshotStoreOrgCensusClient { + return { + organization: { + findMany: async (args) => { + calls?.push(args); + return rows; + }, + }, + }; +} + +function build(rows: Row[]) { + return createSnapshotStoreOrgCensus({ replica: fakeClient(rows) }, { autoStart: false }); +} + +describe("snapshot store org census", () => { + it("reflects a loaded set of overrides", async () => { + const census = build([ + { id: "org_a", featureFlags: { snapshotStoreOrgMode: "redis-read" } }, + { id: "org_b", featureFlags: { snapshotStoreOrgMode: "dual-write" } }, + { id: "org_c", featureFlags: { snapshotStoreOrgMode: "redis-only" } }, + ]); + + await census.refresh(); + + expect(census.anyOrgReadEnabled()).toBe(true); + expect(census.anyOrgRedisOnly()).toBe(true); + expect(census.isCohortMember("org_a")).toBe(true); + expect(census.isCohortMember("org_b")).toBe(true); + expect(census.isCohortMember("org_c")).toBe(true); + expect(census.isCohortMember("org_d")).toBe(false); + }); + + it("reports an empty census once loaded with no active override", async () => { + const census = build([ + { id: "org_a", featureFlags: { snapshotStoreOrgMode: "off" } }, + { id: "org_b", featureFlags: null }, + ]); + + await census.refresh(); + + expect(census.anyOrgReadEnabled()).toBe(false); + expect(census.anyOrgRedisOnly()).toBe(false); + expect(census.isCohortMember("org_a")).toBe(false); + expect(census.isCohortMember("org_b")).toBe(false); + }); + + it("errs toward routing before the first successful load (cold)", () => { + const census = build([{ id: "org_a", featureFlags: { snapshotStoreOrgMode: "redis-only" } }]); + + // Deliberately NOT refreshed: this is the cold window. + expect(census.anyOrgReadEnabled()).toBe(true); + expect(census.anyOrgRedisOnly()).toBe(false); + expect(census.isCohortMember("org_a")).toBe(false); + }); + + it("keeps the last-good snapshot when a later load fails", async () => { + let calls = 0; + const client: SnapshotStoreOrgCensusClient = { + organization: { + findMany: async () => { + calls += 1; + if (calls === 1) { + return [{ id: "org_a", featureFlags: { snapshotStoreOrgMode: "redis-only" } }]; + } + throw new Error("control plane unreachable"); + }, + }, + }; + const census = createSnapshotStoreOrgCensus({ replica: client }, { autoStart: false }); + + await census.refresh(); + expect(census.anyOrgReadEnabled()).toBe(true); + expect(census.anyOrgRedisOnly()).toBe(true); + expect(census.isCohortMember("org_a")).toBe(true); + + // A failing reload must not throw and must not revert to cold defaults. + await expect(census.refresh()).resolves.toBeUndefined(); + expect(census.anyOrgReadEnabled()).toBe(true); + expect(census.anyOrgRedisOnly()).toBe(true); + expect(census.isCohortMember("org_a")).toBe(true); + }); + + it("bounds the query to orgs that have EITHER the mode or the latch key present", async () => { + const calls: FindManyArgs[] = []; + const census = createSnapshotStoreOrgCensus( + { + replica: fakeClient( + [{ id: "org_a", featureFlags: { snapshotStoreOrgMode: "redis-only" } }], + calls + ), + }, + { autoStart: false } + ); + + await census.refresh(); + + expect(calls).toHaveLength(1); + expect(calls[0].where.OR.map((clause) => clause.featureFlags.path)).toEqual([ + ["snapshotStoreOrgMode"], + ["snapshotStoreOrgEverEnabled"], + ]); + // The WHERE only bounds rows; classification is unaffected. + expect(census.isCohortMember("org_a")).toBe(true); + expect(census.anyOrgRedisOnly()).toBe(true); + }); + + it("counts a dual-write-only org as a cohort member without enabling reads", async () => { + const census = build([{ id: "org_a", featureFlags: { snapshotStoreOrgMode: "dual-write" } }]); + + await census.refresh(); + + expect(census.anyOrgReadEnabled()).toBe(false); + expect(census.anyOrgRedisOnly()).toBe(false); + expect(census.isCohortMember("org_a")).toBe(true); + }); +}); + +describe("snapshot store org census — autoStart host gate", () => { + it("never polls when no Redis host is configured, even in production", () => { + expect(defaultCensusAutoStart(undefined, "production")).toBe(false); + expect(defaultCensusAutoStart("", "production")).toBe(false); + }); + + it("polls only when configured and outside test", () => { + expect(defaultCensusAutoStart("snap-redis", "production")).toBe(true); + expect(defaultCensusAutoStart("snap-redis", "test")).toBe(false); + }); + + it("does not issue a query when built with a disabled autoStart", async () => { + const calls: FindManyArgs[] = []; + const census = createSnapshotStoreOrgCensus( + { replica: fakeClient([{ id: "org_a", featureFlags: null }], calls) }, + { autoStart: defaultCensusAutoStart(undefined, "production"), intervalMs: 5 } + ); + + await new Promise((resolve) => setTimeout(resolve, 30)); + + expect(calls).toHaveLength(0); + census.stop(); + }); +}); + +describe("snapshot store org census — definite ever-enabled set", () => { + // A: latched but dialled back to off. B: at redis-read and latched. C: has the mode key but off, + // never latched. D: not returned by the query at all. + const rows = [ + { + id: "org_a", + featureFlags: { snapshotStoreOrgMode: "off", snapshotStoreOrgEverEnabled: true }, + }, + { + id: "org_b", + featureFlags: { snapshotStoreOrgMode: "redis-read", snapshotStoreOrgEverEnabled: true }, + }, + { id: "org_c", featureFlags: { snapshotStoreOrgMode: "off" } }, + ]; + + it("marks only orgs outside the ever-enabled set as definitely never enabled", async () => { + const census = build(rows); + + await census.refresh(); + + expect(census.orgDefinitelyNeverEnabled("org_a")).toBe(false); + expect(census.orgDefinitelyNeverEnabled("org_b")).toBe(false); + expect(census.orgDefinitelyNeverEnabled("org_c")).toBe(true); + // Never returned by the query, so it is not in the set: definitely never enabled. + expect(census.orgDefinitelyNeverEnabled("org_d")).toBe(true); + }); + + it("treats a latched-but-off org as ever-enabled yet outside the read cohort", async () => { + const census = build(rows); + + await census.refresh(); + + expect(census.orgDefinitelyNeverEnabled("org_a")).toBe(false); + expect(census.isCohortMember("org_a")).toBe(false); + // org_a is off; only org_b (redis-read) enables reads. + expect(census.anyOrgReadEnabled()).toBe(true); + expect(census.isCohortMember("org_b")).toBe(true); + }); + + it("reports nobody as definitely-never before the first load (cold)", () => { + const census = build(rows); + + // Deliberately NOT refreshed: not-definite, so the caller must not skip anyone. + expect(census.orgDefinitelyNeverEnabled("org_a")).toBe(false); + expect(census.orgDefinitelyNeverEnabled("org_c")).toBe(false); + expect(census.orgDefinitelyNeverEnabled("org_d")).toBe(false); + }); + + it("keeps the last-good ever-enabled set when a later load fails", async () => { + let calls = 0; + const client: SnapshotStoreOrgCensusClient = { + organization: { + findMany: async () => { + calls += 1; + if (calls === 1) return rows; + throw new Error("control plane unreachable"); + }, + }, + }; + const census = createSnapshotStoreOrgCensus({ replica: client }, { autoStart: false }); + + await census.refresh(); + expect(census.orgDefinitelyNeverEnabled("org_c")).toBe(true); + expect(census.orgDefinitelyNeverEnabled("org_b")).toBe(false); + + await expect(census.refresh()).resolves.toBeUndefined(); + expect(census.orgDefinitelyNeverEnabled("org_c")).toBe(true); + expect(census.orgDefinitelyNeverEnabled("org_b")).toBe(false); + }); +}); diff --git a/apps/webapp/test/snapshotStoreOrgEverEnabled.test.ts b/apps/webapp/test/snapshotStoreOrgEverEnabled.test.ts new file mode 100644 index 00000000000..0dc40b96328 --- /dev/null +++ b/apps/webapp/test/snapshotStoreOrgEverEnabled.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { + clearedOrgFlagsPreservingLatch, + FEATURE_FLAG, + stampSnapshotStoreOrgEverEnabled, + withoutOrgForbiddenSnapshotKeys, +} from "~/v3/featureFlags"; + +const MODE = FEATURE_FLAG.snapshotStoreOrgMode; +const LATCH = FEATURE_FLAG.snapshotStoreOrgEverEnabled; + +describe("stampSnapshotStoreOrgEverEnabled (one-way per-org latch)", () => { + it("latches true when the resulting dial moves past off", () => { + for (const mode of ["dual-write", "redis-read", "redis-only"]) { + const stamped = stampSnapshotStoreOrgEverEnabled(null, { [MODE]: mode }); + expect(stamped[LATCH], mode).toBe(true); + } + }); + + it("keeps the latch true when the org is set back to off (one-way)", () => { + const stamped = stampSnapshotStoreOrgEverEnabled({ [LATCH]: true }, { [MODE]: "off" }); + expect(stamped[LATCH]).toBe(true); + }); + + it("carries an existing latch forward on an unrelated save that omits the dial", () => { + const stamped = stampSnapshotStoreOrgEverEnabled({ [LATCH]: true }, { someOther: "flag" }); + expect(stamped[LATCH]).toBe(true); + }); + + it("leaves the latch absent (never false) when off and never previously enabled", () => { + const stamped = stampSnapshotStoreOrgEverEnabled(null, { [MODE]: "off" }); + expect(LATCH in stamped).toBe(false); + }); + + it("leaves the latch absent when the dial is omitted and never previously enabled", () => { + const stamped = stampSnapshotStoreOrgEverEnabled(null, { someOther: "flag" }); + expect(LATCH in stamped).toBe(false); + }); + + it("is stripped from an operator-supplied org save payload", () => { + expect(withoutOrgForbiddenSnapshotKeys({ [LATCH]: false, [MODE]: "off" })).toEqual({ + [MODE]: "off", + }); + }); +}); + +describe("clearedOrgFlagsPreservingLatch (clear-all keeps the one-way latch)", () => { + it("preserves the latch when the org was ever enabled", () => { + expect(clearedOrgFlagsPreservingLatch({ [LATCH]: true, [MODE]: "off" })).toEqual({ + [LATCH]: true, + }); + }); + + it("wipes to null when the org never latched", () => { + expect(clearedOrgFlagsPreservingLatch({ [MODE]: "off" })).toBeNull(); + expect(clearedOrgFlagsPreservingLatch({})).toBeNull(); + expect(clearedOrgFlagsPreservingLatch(null)).toBeNull(); + }); + + it("never treats a false latch as latched", () => { + expect(clearedOrgFlagsPreservingLatch({ [LATCH]: false })).toBeNull(); + }); +}); diff --git a/apps/webapp/test/snapshotSweepRunner.test.ts b/apps/webapp/test/snapshotSweepRunner.test.ts new file mode 100644 index 00000000000..c1246ad53ad --- /dev/null +++ b/apps/webapp/test/snapshotSweepRunner.test.ts @@ -0,0 +1,188 @@ +import { redisTest } from "@internal/testcontainers"; +import { expect } from "vitest"; +import { createRedisClient } from "@internal/redis"; +import { buildSnapshotSweepRunner } from "~/v3/snapshotSweepRunner.server"; + +const LOCK_KEY = "snapshot-sweep:lock"; +const CLEAN = { scanned: 1, expired: 0, deleted: 0, skipped: 1, partial: false }; + +function opts() { + return { deadline: Date.now() + 10_000, signal: new AbortController().signal }; +} + +redisTest("reports completed and releases the lock", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => CLEAN, + lockTtlMs: 60_000, + }); + + expect((await runner(opts())).outcome).toBe("completed"); + expect(await client.get(LOCK_KEY)).toBeNull(); + } finally { + await client.quit(); + } +}); + +redisTest("reports partial when the pass truncates", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => ({ ...CLEAN, partial: true }), + lockTtlMs: 60_000, + }); + + const result = await runner(opts()); + expect(result.outcome).toBe("partial"); + expect(result.counts).toMatchObject({ partial: true }); + } finally { + await client.quit(); + } +}); + +redisTest("skips without running when the lock is held", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + await client.set(LOCK_KEY, "someone-else", "PX", 60_000); + let ran = false; + + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => { + ran = true; + return CLEAN; + }, + lockTtlMs: 60_000, + }); + + expect((await runner(opts())).outcome).toBe("skipped_locked"); + expect(ran).toBe(false); + expect(await client.get(LOCK_KEY)).toBe("someone-else"); + } finally { + await client.quit(); + } +}); + +redisTest("reports failed and still releases its own lock", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => { + throw new Error("redis down mid-pass"); + }, + lockTtlMs: 60_000, + }); + + // Resolving is deliberate: the worker reschedules a cron job on acknowledge as well as on the + // dead-letter path, so a failure needs no throw to keep the chain alive. + expect((await runner(opts())).outcome).toBe("failed"); + expect(await client.get(LOCK_KEY)).toBeNull(); + } finally { + await client.quit(); + } +}); + +redisTest("an overrun pass cannot delete a successor's lock", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const runner = buildSnapshotSweepRunner({ + client, + fence: () => "first-pass", + // Stand in for the lock expiring mid-pass and a successor claiming it. + sweep: async () => { + await client.set(LOCK_KEY, "successor", "PX", 60_000); + return CLEAN; + }, + lockTtlMs: 60_000, + }); + + await runner(opts()); + + expect(await client.get(LOCK_KEY)).toBe("successor"); + } finally { + await client.quit(); + } +}); + +redisTest("reports aborted when shutdown cancels the pass", async ({ redisOptions }) => { + const client = createRedisClient(redisOptions); + try { + const controller = new AbortController(); + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => { + controller.abort(); + return CLEAN; + }, + lockTtlMs: 60_000, + }); + + const result = await runner({ deadline: Date.now() + 10_000, signal: controller.signal }); + expect(result.outcome).toBe("aborted"); + expect(await client.get(LOCK_KEY)).toBeNull(); + } finally { + await client.quit(); + } +}); + +function flakyEvalClient(real: RedisClient, failEvalTimes: number): RedisClient { + let fails = 0; + return new Proxy(real, { + get(target, prop, receiver) { + if (prop === "eval") { + return (...args: unknown[]) => { + if (fails < failEvalTimes) { + fails += 1; + return Promise.reject(new Error("simulated release eval failure")); + } + return (target.eval as (...a: unknown[]) => unknown)(...args); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }) as unknown as RedisClient; +} + +redisTest("retries a transiently-failing release and frees the lock", async ({ redisOptions }) => { + const real = createRedisClient(redisOptions); + const client = flakyEvalClient(real, 2); // first two release attempts fail, third succeeds + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => CLEAN, + lockTtlMs: 60_000, + releaseRetryDelaysMs: [0, 0, 0], + }); + + expect((await runner(opts())).outcome).toBe("completed"); + expect(await real.get(LOCK_KEY)).toBeNull(); // released after the retries, not left to TTL + } finally { + await real.quit(); + } +}); + +redisTest( + "leaves the lock to its TTL only when every release attempt fails", + async ({ redisOptions }) => { + const real = createRedisClient(redisOptions); + const client = flakyEvalClient(real, 999); // release can never succeed + try { + const runner = buildSnapshotSweepRunner({ + client, + sweep: async () => CLEAN, + lockTtlMs: 60_000, + releaseRetryDelaysMs: [0], + }); + + expect((await runner(opts())).outcome).toBe("completed"); // runner never throws on a release failure + expect(await real.get(LOCK_KEY)).not.toBeNull(); // still held; the TTL is the backstop + } finally { + await real.quit(); + } + } +); diff --git a/docker/config/alerts/snapshot-store.yml b/docker/config/alerts/snapshot-store.yml new file mode 100644 index 00000000000..081afcbaea5 --- /dev/null +++ b/docker/config/alerts/snapshot-store.yml @@ -0,0 +1,68 @@ +# Alerting rules for the execution-snapshot store. +# +# These are paging rules, not dashboard panels. A sustained append failure burns an attempt number +# on every state transition, so runs exhaust their retry budget from an infrastructure fault rather +# than from anything the task did. That is silent to the customer until their run fails for good. +# The exported names carry a deployment-dependent prefix (locally `triggerdotdev_`) that is added +# somewhere in the OpenTelemetry pipeline, not by the instrument names in the code. Matching on the +# suffix keeps these rules correct wherever that prefix differs, and an alert that silently matches +# nothing is worse than no alert. +groups: + - name: snapshot-store + interval: 30s + rules: + - alert: SnapshotStoreAppendFailing + # Any sustained failure at all, not a rate threshold: one failed append per minute still + # burns one attempt per minute on the runs it touches. + expr: sum(rate({__name__=~".*run_engine_snapshot_store_append_failed(_total)?"}[5m])) > 0 + for: 10m + labels: + severity: page + annotations: + summary: "Execution-snapshot appends to Redis are failing" + description: >- + Appends have failed continuously for 10 minutes. Each failure burns an attempt number on + the run it touches, so runs will exhaust their retry budget from infrastructure rather + than task failure. Turning the snapshotStoreMode flag down to off stops new runs + entering Redis but leaves resident runs mirroring, so the failures continue until they + finish; the snapshotStoreHalt flag stops every append at once, at the cost of freezing + resident heads and requiring a resync. Both take effect without a deploy. Then + investigate the Redis endpoint. + + - alert: SnapshotStoreAppendForked + # A fork is a divergence that has already happened: the head moved under an append that was + # refused, so Redis and Postgres now disagree about that run's latest snapshot. With a run's + # store fixed at birth the writer set per run is stable, so this is a lost append or a + # genuine concurrent writer, not steady-state noise. + # + # A fork now enqueues a repair automatically, and the repair re-derives the head from + # Postgres. So this alert means "divergence happened", not "divergence persists": it should + # clear on its own. It pages because a fork that does NOT clear is a stuck repair, and + # because the entries lost with the append are not recovered even when the head is. + expr: sum(rate({__name__=~".*run_engine_snapshot_store_write_total(_total)?",outcome="forked"}[5m])) > 0 + for: 5m + labels: + severity: page + annotations: + summary: "Execution-snapshot appends are being refused as forked" + description: >- + One or more runs had a Redis head that disagreed with Postgres. A repair was enqueued + automatically for each, so first check whether the rate is falling: a fork that clears + itself needs no action. If it is not falling, the repair is not running, and that is the + thing to investigate. Below redis-only Postgres is authoritative, so turning the + snapshotStoreMode flag down to off stops new runs entering Redis while you look. Do NOT + halt: that disables the repair and freezes every resident run's head instead of just the + forked ones. Note that a repaired run has a correct head but is missing the entries lost + with the append, so its keyspace is marked and window reads fall back to Postgres. + + - alert: SnapshotStoreSweepNotCompleting + # The sweep is the only reaper for orphaned keyspaces. Its absence is silent by nature. + expr: sum(increase({__name__=~".*run_engine_snapshot_store_sweep_pass_total(_total)?",outcome="completed"}[24h])) == 0 + for: 1h + labels: + severity: ticket + annotations: + summary: "No execution-snapshot sweep pass completed in 24 hours" + description: >- + The sweep runs every 6 hours by default, so 24 hours with no completed pass means the + job is not running, is failing, or is losing its lock every pass. diff --git a/docker/config/prometheus.yml b/docker/config/prometheus.yml index 19e681de94a..8e5b37a8b21 100644 --- a/docker/config/prometheus.yml +++ b/docker/config/prometheus.yml @@ -5,6 +5,11 @@ global: scrape_interval: 15s evaluation_interval: 15s +# Paging rules live in the repo so they are reviewable and can be checked locally with +# `promtool check rules`. Mounted read-only by docker-compose.extras.yml. +rule_files: + - /etc/prometheus/alerts/*.yml + scrape_configs: # Scrape OpenTelemetry Collector's Prometheus exporter # This includes all OTel metrics (batch queue, fair queue, etc.) diff --git a/docker/config/redis-cluster-init.sh b/docker/config/redis-cluster-init.sh new file mode 100755 index 00000000000..6c93a98bf70 --- /dev/null +++ b/docker/config/redis-cluster-init.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Brings up a six-node Redis cluster (3 masters, 3 replicas) inside ONE container. +# +# One container, not six, for two reasons that both come from Docker on macOS: +# +# 1. Cluster nodes gossip with each other using the address they ADVERTISE. Six separate +# containers each advertising 127.0.0.1 would each be talking to themselves, and the cluster +# never forms. Sharing one network namespace makes 127.0.0.1 mean the same thing to all six. +# 2. A client on the host follows MOVED redirects to that same advertised address, so the ports +# published below resolve correctly from outside. +# +# Each node gets its own directory: six nodes sharing one working directory fight over nodes.conf +# and all but the first fail to start, silently, because they are daemonised. +set -e + +NODES="1 2 3 4 5 6" + +for i in $NODES; do + PORT=$((7000 + i)) + BUS=$((17000 + i)) + mkdir -p "/data/$PORT" + redis-server \ + --port "$PORT" \ + --cluster-enabled yes \ + --cluster-node-timeout 5000 \ + --cluster-announce-ip 127.0.0.1 \ + --cluster-announce-port "$PORT" \ + --cluster-announce-bus-port "$BUS" \ + --cluster-config-file "nodes-$PORT.conf" \ + --dir "/data/$PORT" \ + --protected-mode no \ + --appendonly no \ + --save '' \ + --logfile "/data/$PORT/redis.log" \ + --daemonize yes +done + +# Daemonised servers report failures only to their own log, so check before forming the cluster. +sleep 5 +for i in $NODES; do + PORT=$((7000 + i)) + if ! redis-cli -p "$PORT" ping >/dev/null 2>&1; then + echo "node $PORT failed to start:" + tail -20 "/data/$PORT/redis.log" + exit 1 + fi +done + +# Idempotent: a restart with a populated /data already has slots assigned, so skip the create. +if redis-cli -p 7001 cluster info | grep -q "cluster_state:ok"; then + echo "cluster already formed" +else + redis-cli --cluster create \ + 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 \ + 127.0.0.1:7004 127.0.0.1:7005 127.0.0.1:7006 \ + --cluster-replicas 1 --cluster-yes +fi + +redis-cli -p 7001 cluster info | grep -E "cluster_state|cluster_size" +exec tail -f /dev/null diff --git a/docker/docker-compose.extras.yml b/docker/docker-compose.extras.yml index cf16272dcc5..ed2eaabcc00 100644 --- a/docker/docker-compose.extras.yml +++ b/docker/docker-compose.extras.yml @@ -14,6 +14,7 @@ name: triggerdotdev-docker volumes: + redis-cluster-data: prometheus-data: grafana-data: @@ -95,6 +96,7 @@ services: restart: always volumes: - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./config/alerts:/etc/prometheus/alerts:ro - prometheus-data:/prometheus ports: - "${PROMETHEUS_HOST_PORT:-9090}:9090" @@ -122,3 +124,39 @@ services: - app_network depends_on: - prometheus + + # Six-node Redis cluster (3 masters, 3 replicas) for validating the execution-snapshot store. + # The store's single-slot guarantee only means anything against a cluster-mode endpoint, and the + # orphan sweeper's per-master fan-out is unreachable on a single node. + # + # RUN_ENGINE_SNAPSHOT_STORE_REDIS_HOST=127.0.0.1 + # RUN_ENGINE_SNAPSHOT_STORE_REDIS_PORT=7001 + # RUN_ENGINE_SNAPSHOT_STORE_REDIS_CLUSTER_MODE_ENABLED=1 + # + # ioredis discovers the other five nodes from that one seed. + redis-cluster: + container_name: ${CONTAINER_PREFIX:-}redis-cluster + image: redis:7.2@sha256:74566c6910d13ae61e7ce73ebd3127438a1fe805b309b097c323142719ec8a5b + restart: always + volumes: + - ./config/redis-cluster-init.sh:/init.sh:ro + - redis-cluster-data:/data + command: sh /init.sh + ports: + - "${REDIS_CLUSTER_PORT_1:-7001}:7001" + - "${REDIS_CLUSTER_PORT_2:-7002}:7002" + - "${REDIS_CLUSTER_PORT_3:-7003}:7003" + - "${REDIS_CLUSTER_PORT_4:-7004}:7004" + - "${REDIS_CLUSTER_PORT_5:-7005}:7005" + - "${REDIS_CLUSTER_PORT_6:-7006}:7006" + - "17001:17001" + - "17002:17002" + - "17003:17003" + - "17004:17004" + - "17005:17005" + - "17006:17006" + healthcheck: + test: ["CMD-SHELL", "redis-cli -p 7001 cluster info | grep -q cluster_state:ok"] + interval: 5s + timeout: 3s + retries: 20 diff --git a/docker/otel-collector-config.yaml b/docker/otel-collector-config.yaml index b574c1295df..52be68d82e4 100644 --- a/docker/otel-collector-config.yaml +++ b/docker/otel-collector-config.yaml @@ -12,6 +12,15 @@ processors: exporters: logging: verbosity: normal + # Serves the collected metrics on :8889/metrics, which docker-compose.extras.yml publishes and + # config/prometheus.yml already scrapes. Without this pipeline that scrape target returns nothing, + # so no OTel metric is visible locally. + prometheus: + endpoint: 0.0.0.0:8889 + # Keeps the metric names as OpenTelemetry emitted them, so a rule written against + # run_engine_snapshot_store_* matches without a namespace guess. + resource_to_telemetry_conversion: + enabled: false otlphttp: endpoint: "http://host.docker.internal:3030/otel" compression: none @@ -26,3 +35,7 @@ service: receivers: [otlp] processors: [batch] exporters: [otlphttp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] diff --git a/internal-packages/redis/package.json b/internal-packages/redis/package.json index 14cf7f33d61..eff0a779dd7 100644 --- a/internal-packages/redis/package.json +++ b/internal-packages/redis/package.json @@ -10,6 +10,8 @@ "@trigger.dev/core": "workspace:*" }, "scripts": { - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "test": "vitest", + "test:watch": "vitest" } } diff --git a/internal-packages/redis/src/cluster.test.ts b/internal-packages/redis/src/cluster.test.ts new file mode 100644 index 00000000000..71d2989b22f --- /dev/null +++ b/internal-packages/redis/src/cluster.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { Cluster, createRedisClusterClient, defaultReconnectOnError } from "./index.js"; + +// Port 6399 is deliberately closed: these assertions are about the client the factory builds, not +// about a connection. +const NODES = [{ host: "127.0.0.1", port: 6399 }]; + +type InnerOptions = { + options: { + redisOptions?: { + reconnectOnError?: unknown; + maxRetriesPerRequest?: number; + retryStrategy?: unknown; + keyPrefix?: string; + }; + }; +}; + +function innerOptionsOf(client: Cluster) { + return (client as unknown as InnerOptions).options.redisOptions; +} + +describe("createRedisClusterClient", () => { + it("returns a Cluster instance", async () => { + const client = createRedisClusterClient({ nodes: NODES }); + try { + expect(client).toBeInstanceOf(Cluster); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("installs defaultReconnectOnError on the inner per-node options", async () => { + const client = createRedisClusterClient({ nodes: NODES }); + try { + expect(innerOptionsOf(client)?.reconnectOnError).toBe(defaultReconnectOnError); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("carries the retry defaults onto the inner options", async () => { + const client = createRedisClusterClient({ nodes: NODES }); + try { + const inner = innerOptionsOf(client); + expect(inner?.maxRetriesPerRequest).toBeTypeOf("number"); + expect(inner?.retryStrategy).toBeTypeOf("function"); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("lets caller redisOptions override the defaults", async () => { + const client = createRedisClusterClient({ + nodes: NODES, + redisOptions: { keyPrefix: "engine:", maxRetriesPerRequest: 3 }, + }); + try { + const inner = innerOptionsOf(client); + expect(inner?.keyPrefix).toBe("engine:"); + expect(inner?.maxRetriesPerRequest).toBe(3); + } finally { + await client.quit().catch(() => undefined); + } + }); + + it("keeps mapping READONLY, LOADING and UNBLOCKED to a reconnect-and-retry", () => { + expect(defaultReconnectOnError(new Error("READONLY against a read only replica"))).toBe(2); + expect(defaultReconnectOnError(new Error("LOADING Redis is loading the dataset"))).toBe(2); + expect(defaultReconnectOnError(new Error("UNBLOCKED force unblock"))).toBe(2); + expect(defaultReconnectOnError(new Error("ERR unknown command"))).toBe(false); + }); +}); diff --git a/internal-packages/redis/src/clusterFailFast.test.ts b/internal-packages/redis/src/clusterFailFast.test.ts new file mode 100644 index 00000000000..463da45d7d0 --- /dev/null +++ b/internal-packages/redis/src/clusterFailFast.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { createRedisClusterClient } from "./index.js"; + +// The offline queue on a Cluster client is a CLUSTER-level option. Setting it on the inner +// per-node redisOptions leaves the cluster queueing commands while it cannot refresh its slot +// cache, so a command issued during an outage waits instead of failing and the caller hangs. +describe("cluster client against an unreachable cluster", () => { + it("rejects a command rather than queueing it", async () => { + const client = createRedisClusterClient({ + // Nothing listens here. + nodes: [{ host: "127.0.0.1", port: 6391 }], + redisOptions: { commandTimeout: 300 }, + failFast: true, + }); + + const started = Date.now(); + await expect(client.get("anything")).rejects.toThrow(); + const elapsed = Date.now() - started; + + expect(elapsed).toBeLessThan(5000); + + client.disconnect(); + }); +}); diff --git a/internal-packages/redis/src/failFast.test.ts b/internal-packages/redis/src/failFast.test.ts new file mode 100644 index 00000000000..1a4bdaa1c83 --- /dev/null +++ b/internal-packages/redis/src/failFast.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { createRedisClient } from "./index.js"; + +// A snapshot-store append sits on a request path. With the offline queue enabled and no command +// timeout, a command issued while the endpoint is unreachable waits for a reconnect instead of +// failing, so the request hangs rather than falling back to Postgres. +describe("fail-fast options against an unreachable endpoint", () => { + it("rejects rather than hanging when the offline queue is off and a timeout is set", async () => { + const client = createRedisClient({ + // Nothing listens here. + host: "127.0.0.1", + port: 6390, + enableOfflineQueue: false, + commandTimeout: 300, + lazyConnect: true, + retryStrategy: () => null, + }); + + const started = Date.now(); + await expect(client.get("anything")).rejects.toThrow(); + const elapsed = Date.now() - started; + + // Generous, but far below the indefinite wait the offline queue produces. + expect(elapsed).toBeLessThan(3000); + + client.disconnect(); + }); +}); diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts index 622efe613fc..0c0f380dab8 100644 --- a/internal-packages/redis/src/index.ts +++ b/internal-packages/redis/src/index.ts @@ -1,4 +1,10 @@ -import { type Cluster, Redis, type RedisOptions } from "ioredis"; +import { + Redis, + type Cluster, + type ClusterNode, + type ClusterOptions, + type RedisOptions, +} from "ioredis"; import { Logger } from "@trigger.dev/core/logger"; export { @@ -85,3 +91,55 @@ export function createRedisClient( return client; } + +export type RedisClusterClientOptions = { + nodes: ClusterNode[]; + clusterOptions?: Omit; + redisOptions?: RedisOptions; + /** + * Fail a command while the cluster is unreachable instead of queueing it. For a caller on a + * request path, where a queued command means a hung request rather than a slow one. + */ + failFast?: boolean; +}; + +/** + * Cluster-mode client. `defaultOptions` go on the INNER per-node options, so a role swap gets the + * same reconnect-and-retry treatment a single-node client already gets. + */ +export function createRedisClusterClient( + options: RedisClusterClientOptions, + handlers?: { onError?: (err: Error) => void } +): Cluster { + const client = new Redis.Cluster(options.nodes, { + // The offline queue is a CLUSTER-level setting, separate from the per-node one below. While a + // cluster cannot refresh its slot cache it queues commands here, so a caller that wants a + // failure during an outage rather than a wait has to turn THIS one off. Default stays `true`, + // matching ioredis, so only a caller that asks for it changes behaviour. + ...(options.failFast && { enableOfflineQueue: false }), + ...options.clusterOptions, + redisOptions: { + ...defaultOptions, + ...(options.failFast && { enableOfflineQueue: false }), + ...options.redisOptions, + }, + }); + + if (process.env.VITEST) { + client.on("error", () => {}); + return client; + } + + client.on("error", (error) => { + if (handlers?.onError) { + handlers.onError(error); + } else { + logger.error(`Redis cluster client error:`, { + error, + keyPrefix: options.redisOptions?.keyPrefix, + }); + } + }); + + return client; +} diff --git a/internal-packages/redis/vitest.config.ts b/internal-packages/redis/vitest.config.ts new file mode 100644 index 00000000000..e07f05e842b --- /dev/null +++ b/internal-packages/redis/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.test.ts"], + globals: true, + isolate: true, + testTimeout: 10_000, + }, +}); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index 0ff2b8068de..17573b18672 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -3,6 +3,7 @@ import { type Meter, type Tracer, type Counter, + type Histogram, getMeter, startSpan, trace, @@ -86,21 +87,29 @@ import { RaceSimulationSystem } from "./systems/raceSimulationSystem.js"; import { RunAttemptSystem } from "./systems/runAttemptSystem.js"; import { NoopPendingVersionRunIdLookup } from "./services/pendingVersionLookup.js"; import type { SystemResources } from "./systems/systems.js"; -import { type RunStore, PostgresRunStore } from "@internal/run-store"; +import { type RunStore, asSnapshotMirrorRepair, PostgresRunStore } from "@internal/run-store"; import { type ControlPlaneResolver, PassthroughControlPlaneResolver, } from "./controlPlaneResolver.js"; import { TtlSystem } from "./systems/ttlSystem.js"; import { WaitpointSystem } from "./systems/waitpointSystem.js"; +import { SNAPSHOT_SWEEP_COUNT_FIELDS } from "./types.js"; import type { EngineWorker, HeartbeatTimeouts, ReportableQueue, RunEngineOptions, + SnapshotSweepOutcome, TriggerParams, } from "./types.js"; import { createTtlWorkerCatalog } from "./ttlWorkerCatalog.js"; +import { + DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS, + resolveSnapshotSweepCron, + seedSnapshotSweepOutcomes, + snapshotSweepVisibilityTimeoutMs, +} from "./snapshotSweepSchedule.js"; import { workerCatalog } from "./workerCatalog.js"; import pMap from "p-map"; @@ -112,6 +121,8 @@ export class RunEngine { private logger: Logger; private tracer: Tracer; private meter: Meter; + private snapshotSweepPassCounter?: Counter; + private snapshotSweepCountsHistogram?: Histogram; private snapshotsSinceReplicaMissCounter: Counter; private snapshotsSinceReplicaRetryDelay: { minMs: number; maxMs: number }; private heartbeatTimeouts: HeartbeatTimeouts; @@ -252,7 +263,25 @@ export class RunEngine { ...options.worker.redis, keyPrefix: `${options.worker.redis.keyPrefix}worker:`, }, - catalog: workerCatalog, + catalog: { + ...workerCatalog, + sweepSnapshotOrphans: { + ...workerCatalog.sweepSnapshotOrphans, + // Derived, not fixed. The runner's lock TTL is the budget plus an hour, so a hardcoded + // timeout would sit below the lock once the budget is raised, inverting the ordering the + // fence depends on. + visibilityTimeoutMs: snapshotSweepVisibilityTimeoutMs( + options.snapshotStore?.sweepBudgetMs + ), + cron: resolveSnapshotSweepCron({ + hasRunner: !!options.snapshotStore?.runSweep, + schedule: options.snapshotStore?.sweepSchedule, + fallback: workerCatalog.sweepSnapshotOrphans.cron, + }), + jitterInMs: + options.snapshotStore?.sweepJitterInMs ?? workerCatalog.sweepSnapshotOrphans.jitterInMs, + }, + }, concurrency: options.worker, pollIntervalMs: options.worker.pollIntervalMs, immediatePollIntervalMs: options.worker.immediatePollIntervalMs, @@ -276,6 +305,9 @@ export class RunEngine { repairSnapshot: async ({ payload }) => { await this.#handleRepairSnapshot(payload); }, + sweepSnapshotOrphans: async () => { + await this.#handleSweepSnapshotOrphans(); + }, expireRun: async ({ payload }) => { await this.ttlSystem.expireRun({ runId: payload.runId }); }, @@ -330,6 +362,27 @@ export class RunEngine { this.tracer = options.tracer; this.meter = options.meter ?? getMeter("run-engine"); + // Only when the sweep is actually wired: a deployment that does not use the snapshot store + // should register no series for it at all. + if (options.snapshotStore?.runSweep) { + this.snapshotSweepPassCounter = this.meter.createCounter( + "run_engine.snapshot_store.sweep_pass_total", + { + description: + "Orphan-sweep passes by outcome. A pass that throws emits outcome=failed, so silence is distinguishable from success", + } + ); + + // Seeded at zero so the series EXISTS from boot. Inside this block on purpose: a deployment + // that does not wire the sweep registers no series and cannot alert. + seedSnapshotSweepOutcomes(this.snapshotSweepPassCounter); + + this.snapshotSweepCountsHistogram = this.meter.createHistogram( + "run_engine.snapshot_store.sweep_counts", + { description: "Per-field counts from one orphan-sweep pass" } + ); + } + this.snapshotsSinceReplicaMissCounter = this.meter.createCounter( "run_engine.snapshots_since.replica_miss", { @@ -2481,6 +2534,23 @@ export class RunEngine { }; } + /** + * The append-failure compensator. Shares the stall watchdog's job id AND its availableAt, so the + * two cannot enqueue two repairs for one run and neither can win a race that changes the delay. + */ + async enqueueSnapshotRepair(payload: { + runId: string; + snapshotId: string; + executionStatus: string; + }): Promise { + return this.worker.enqueueOnce({ + id: `repair-in-progress-run:${payload.runId}`, + job: "repairSnapshot", + payload, + availableAt: new Date(Date.now() + this.repairSnapshotTimeoutMs), + }); + } + async #repairRun(runId: string, dryRun: boolean) { const snapshot = await getLatestExecutionSnapshot(this.prisma, runId, this.runStore); @@ -2907,6 +2977,53 @@ export class RunEngine { }); } + /** + * One emitter per pass, in a finally, so a throw is reported rather than silent. The deploy step + * gates dual-write on an observed pass, so an absent metric would read as a clean sweep. + */ + async #handleSweepSnapshotOrphans() { + const runSweep = this.options.snapshotStore?.runSweep; + + if (!runSweep) { + // The cron entry is registered when the engine is constructed, which happens before the + // webapp sets the binding, so an occurrence already queued at boot can arrive unbound. + this.snapshotSweepPassCounter?.add(1, { outcome: "unbound" }); + this.logger.error("sweepSnapshotOrphans ran with no sweep runner bound"); + return; + } + + const budgetMs = this.options.snapshotStore?.sweepBudgetMs ?? DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS; + const controller = new AbortController(); + // The deadline is the sweep's own stopping rule; this is the backstop for a pass that has + // stopped reaching a batch boundary, so the signal is not merely decorative. + const abortAt = globalThis.setTimeout(() => controller.abort(), budgetMs + 60_000); + let outcome: SnapshotSweepOutcome = "failed"; + let counts: Partial> | undefined; + + try { + const result = await runSweep({ + deadline: Date.now() + budgetMs, + signal: controller.signal, + }); + outcome = result.outcome; + counts = result.counts; + } catch (error) { + // Deliberately not rethrown. Both the acknowledge path and the dead-letter path reschedule a + // cron job, so returning here continues the chain; throwing would only add a dead-letter + // entry for every transient blip. The outcome metric is the signal. + this.logger.error("sweepSnapshotOrphans threw", { error }); + } finally { + globalThis.clearTimeout(abortAt); + this.snapshotSweepPassCounter?.add(1, { outcome }); + for (const [field, value] of Object.entries(counts ?? {})) { + // Each field is a metric attribute, so an unrecognised key would mint a time series. + if (typeof value === "number" && SNAPSHOT_SWEEP_COUNT_FIELDS.includes(field as never)) { + this.snapshotSweepCountsHistogram?.record(value, { field }); + } + } + } + } + async #handleRepairSnapshot({ runId, snapshotId, @@ -2917,7 +3034,26 @@ export class RunEngine { executionStatus: string; }) { return await this.runLock.lock("handleRepairSnapshot", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(this.prisma, runId, this.runStore); + const mirror = asSnapshotMirrorRepair(this.runStore); + + // Ahead of the staleness guard below, and for every status, because the two things a repair + // can heal are independent. The queue recovery only applies while this snapshot is still the + // latest, but a mirror missing an entry stays wrong however far the run has moved on, and the + // queue recovery cannot put that entry back. Idempotent, so a snapshot that did land is a + // no-op. + if (mirror) { + const outcome = await mirror.repairRedisHead(runId, snapshotId); + this.logger.log("RunEngine.handleRepairSnapshot mirror", { runId, snapshotId, outcome }); + } + + // Through the mirror's UNDECORATED store: once reads are served from Redis, reading through + // `this.runStore` hands this the same stale head the repair was enqueued to replace, so it + // would decide the snapshot is no longer current and stop. + const latestSnapshot = await getLatestExecutionSnapshot( + this.prisma, + runId, + mirror?.authoritativeStore() ?? this.runStore + ); if (latestSnapshot.id !== snapshotId) { this.logger.log( @@ -2933,7 +3069,6 @@ export class RunEngine { return; } - // Okay, so this means we haven't transitioned to a new status yes, so we need to do something switch (latestSnapshot.executionStatus) { case "EXECUTING": case "EXECUTING_WITH_WAITPOINTS": @@ -2942,7 +3077,8 @@ export class RunEngine { case "QUEUED_EXECUTING": case "RUN_CREATED": case "DELAYED": { - // Do nothing; + // The mirror repair above is the whole repair for these: the run is live and the queue + // needs no correction. return; } case "QUEUED": { diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts new file mode 100644 index 00000000000..a6cbb13be69 --- /dev/null +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.test.ts @@ -0,0 +1,100 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS, + resolveSnapshotSweepCron, + seedSnapshotSweepOutcomes, + SNAPSHOT_SWEEP_SEEDED_OUTCOMES, + snapshotSweepVisibilityTimeoutMs, +} from "./snapshotSweepSchedule.js"; + +const FALLBACK = "0 */6 * * *"; + +describe("resolveSnapshotSweepCron", () => { + it("never schedules without a runner", () => { + expect(resolveSnapshotSweepCron({ hasRunner: false, fallback: FALLBACK })).toBeUndefined(); + expect( + resolveSnapshotSweepCron({ hasRunner: false, schedule: "* * * * *", fallback: FALLBACK }) + ).toBeUndefined(); + }); + + it("uses the fallback when no schedule is supplied", () => { + expect(resolveSnapshotSweepCron({ hasRunner: true, fallback: FALLBACK })).toBe(FALLBACK); + }); + + it("uses the supplied schedule", () => { + expect( + resolveSnapshotSweepCron({ hasRunner: true, schedule: "0 */12 * * *", fallback: FALLBACK }) + ).toBe("0 */12 * * *"); + }); + + it("does not let an empty schedule silently disable the job", () => { + expect(resolveSnapshotSweepCron({ hasRunner: true, schedule: "", fallback: FALLBACK })).toBe( + FALLBACK + ); + expect(resolveSnapshotSweepCron({ hasRunner: true, schedule: " ", fallback: FALLBACK })).toBe( + FALLBACK + ); + }); +}); + +describe("the unconfigured deployment", () => { + // The webapp must omit the whole options block, not pass a runner that reports unbound: a + // registered cron would log an unbound pass every interval on every install not using the store. + it("schedules nothing when the options block is absent", () => { + expect( + resolveSnapshotSweepCron({ hasRunner: false, schedule: "0 */6 * * *", fallback: FALLBACK }) + ).toBeUndefined(); + }); +}); + +describe("snapshotSweepVisibilityTimeoutMs", () => { + // The runner's lock TTL is the budget plus an hour. The delivery window has to stay above it, or + // a redelivery arrives while the previous pass still holds the fence. + const lockTtl = (budget: number) => budget + 60 * 60 * 1000; + + it("stays above the lock TTL at the default budget", () => { + expect(snapshotSweepVisibilityTimeoutMs()).toBeGreaterThan( + lockTtl(DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS) + ); + }); + + it("stays above the lock TTL for a raised budget", () => { + for (const budget of [60_000, 10_800_000, 43_200_000, 86_400_000]) { + expect(snapshotSweepVisibilityTimeoutMs(budget)).toBeGreaterThan(lockTtl(budget)); + } + }); +}); + +describe("seeding the sweep outcome series", () => { + it("creates each seeded outcome at zero", () => { + // `sum(increase(...)) == 0` matches NOTHING when the series is absent, so an alert asking "no + // completed passes in 24h" cannot fire on a sweep that has never run. Seeding at zero is what + // makes that absence visible. + const added: { value: number; attributes: Record }[] = []; + seedSnapshotSweepOutcomes({ + add: (value, attributes) => added.push({ value, attributes }), + }); + + expect(added).toEqual([{ value: 0, attributes: { outcome: "completed" } }]); + }); + + it("seeds every outcome the alerting rules query, or the alert cannot fire", () => { + // The invariant that ties the two together: an outcome an alert filters on must be seeded. + // Without this, removing the seed OR changing the alert's outcome silently disables the alert. + const rules = readFileSync( + resolve(__dirname, "../../../../docker/config/alerts/snapshot-store.yml"), + "utf8" + ); + const sweepRule = rules.slice(rules.indexOf("- alert: SnapshotStoreSweepNotCompleting")); + const queried = [...sweepRule.matchAll(/sweep_pass_total[^}]*outcome="([a-z]+)"/g)].map( + (m) => m[1] + ); + + expect(queried.length).toBeGreaterThan(0); + for (const outcome of queried) { + expect(SNAPSHOT_SWEEP_SEEDED_OUTCOMES).toContain(outcome); + } + }); +}); diff --git a/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts new file mode 100644 index 00000000000..adb328ddcce --- /dev/null +++ b/internal-packages/run-engine/src/engine/snapshotSweepSchedule.ts @@ -0,0 +1,48 @@ +/** + * `undefined` never schedules the job. An empty schedule falls back to the default rather than + * through: an empty string is falsy, so `setupCron` would filter the job out with nothing logged. + */ +export function resolveSnapshotSweepCron(opts: { + hasRunner: boolean; + schedule?: string; + fallback: string; +}): string | undefined { + if (!opts.hasRunner) { + return undefined; + } + return opts.schedule?.trim() ? opts.schedule : opts.fallback; +} + +/** Default budget for one sweep pass, when the caller supplies none. */ +export const DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS = 10_800_000; + +/** + * Keeps the delivery window strictly above the runner's lock TTL, which is the budget plus an hour. + * Two hours of headroom, so a pass that overruns its budget still holds a lock that outlives the + * delivery it belongs to. + */ +export function snapshotSweepVisibilityTimeoutMs(budgetMs?: number): number { + return (budgetMs ?? DEFAULT_SNAPSHOT_SWEEP_BUDGET_MS) + 2 * 60 * 60 * 1000; +} + +/** + * Sweep outcomes whose time series must EXIST from boot, not merely once they first occur. + * + * The alert on the sweep asks whether there have been no completed passes in 24 hours, and + * `sum(increase(...)) == 0` matches nothing at all when the series is absent. That is precisely the + * case the alert exists for: a sweep that has never run. Seeding the counter at zero makes the + * absence visible instead of silent. + * + * Anything the alerting rules query by outcome belongs here. + */ +export const SNAPSHOT_SWEEP_SEEDED_OUTCOMES = ["completed"] as const; + +/** Minimal shape of an OTel counter, so this is testable without a meter. */ +type SeedableCounter = { add(value: number, attributes: Record): void }; + +/** Creates the series named in {@link SNAPSHOT_SWEEP_SEEDED_OUTCOMES} at zero. */ +export function seedSnapshotSweepOutcomes(counter: SeedableCounter): void { + for (const outcome of SNAPSHOT_SWEEP_SEEDED_OUTCOMES) { + counter.add(0, { outcome }); + } +} diff --git a/internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts b/internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts new file mode 100644 index 00000000000..3924a5ef4ae --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/helpers/snapshotRepairEngine.ts @@ -0,0 +1,24 @@ +// A minimal engine for asserting the append-failure repair binding. No decorator and no Redis +// snapshot store: the property under test is the job id's dedupe, which lives on the engine. +import { trace } from "@internal/tracing"; + +export function engineOptionsForSnapshotRepair(prisma: unknown, redisOptions: unknown) { + return { + prisma, + worker: { redis: redisOptions, workers: 1, tasksPerWorker: 10, pollIntervalMs: 100 }, + queue: { + redis: redisOptions, + masterQueueConsumersDisabled: true, + processWorkerQueueDebounceMs: 50, + }, + runLock: { redis: redisOptions }, + machines: { + defaultMachine: "small-1x" as const, + machines: { + "small-1x": { name: "small-1x" as const, cpu: 0.5, memory: 0.5, centsPerMs: 0.0001 }, + }, + baseCostInCents: 0.0001, + }, + tracer: trace.getTracer("test", "0.0.0"), + }; +} diff --git a/internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts b/internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts new file mode 100644 index 00000000000..ba14c58034e --- /dev/null +++ b/internal-packages/run-engine/src/engine/tests/snapshotRepairBinding.test.ts @@ -0,0 +1,48 @@ +import { containerTest } from "@internal/testcontainers"; +import { expect } from "vitest"; +import { RunEngine } from "../index.js"; +import { engineOptionsForSnapshotRepair } from "./helpers/snapshotRepairEngine.js"; + +containerTest( + "a second repair for the same run enqueues no second job", + async ({ prisma, redisOptions }) => { + const engine = new RunEngine(engineOptionsForSnapshotRepair(prisma, redisOptions) as never); + + try { + const payload = { + runId: "run_repair_dedupe", + snapshotId: "snap_1", + executionStatus: "EXECUTING", + }; + + // The stall watchdog uses this same job id, so the two compensators must collapse to one. + expect(await engine.enqueueSnapshotRepair(payload)).toBe(true); + expect(await engine.enqueueSnapshotRepair({ ...payload, snapshotId: "snap_2" })).toBe(false); + } finally { + await engine.quit(); + } + } +); + +containerTest("a different run gets its own repair job", async ({ prisma, redisOptions }) => { + const engine = new RunEngine(engineOptionsForSnapshotRepair(prisma, redisOptions) as never); + + try { + expect( + await engine.enqueueSnapshotRepair({ + runId: "run_a", + snapshotId: "snap_a", + executionStatus: "EXECUTING", + }) + ).toBe(true); + expect( + await engine.enqueueSnapshotRepair({ + runId: "run_b", + snapshotId: "snap_b", + executionStatus: "EXECUTING", + }) + ).toBe(true); + } finally { + await engine.quit(); + } +}); diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts index 71dcc424a1f..456a88460bd 100644 --- a/internal-packages/run-engine/src/engine/types.ts +++ b/internal-packages/run-engine/src/engine/types.ts @@ -50,6 +50,31 @@ export type CrossSeamGuardHook = (input: { routeKind: "MANUAL" | "DATETIME" | "RESUME_TOKEN" | "IDEMPOTENCY_REUSE" | "RUN"; }) => Promise; +export type SnapshotSweepOutcome = + | "completed" + | "partial" + | "skipped_locked" + | "failed" + | "unbound" + | "aborted"; + +export const SNAPSHOT_SWEEP_COUNT_FIELDS = [ + "scanned", + "expired", + "deleted", + "skipped", + "pendingDeletion", + // Keyspaces the pass could not read. Reported because containment without a number is + // indistinguishable from nothing going wrong: a run that fails every pass leaks forever and the + // only other evidence is one log line per pass. + "failed", + "nodes", + "partial", +] as const; + +/** Derived from the list above, so the runtime filter and the type cannot drift apart. */ +type SnapshotSweepCountField = (typeof SNAPSHOT_SWEEP_COUNT_FIELDS)[number]; + export type RunEngineOptions = { prisma: PrismaClient; readOnlyPrisma?: PrismaReplicaClient; @@ -166,6 +191,22 @@ export type RunEngineOptions = { randomize?: boolean; }; }; + /** + * The execution-snapshot orphan sweep. The engine owns scheduling only: the webapp owns what + * runs, because a pass needs a run store and its own Redis client and the engine opens neither. + */ + snapshotStore?: { + /** Bounded on purpose: both fields become metric attributes, so each value is a time series. */ + runSweep?: (opts: { deadline: number; signal: AbortSignal }) => Promise<{ + outcome: SnapshotSweepOutcome; + counts?: Partial>; + }>; + /** Cron. Absent or empty falls back to the catalog default. */ + sweepSchedule?: string; + sweepJitterInMs?: number; + /** Ceiling on one pass. Must stay below the job's visibility timeout. */ + sweepBudgetMs?: number; + }; debounce?: { redis?: RedisOptions; /** diff --git a/internal-packages/run-engine/src/engine/workerCatalog.ts b/internal-packages/run-engine/src/engine/workerCatalog.ts index d9000901e38..6b99f631c94 100644 --- a/internal-packages/run-engine/src/engine/workerCatalog.ts +++ b/internal-packages/run-engine/src/engine/workerCatalog.ts @@ -109,4 +109,19 @@ export const workerCatalog = { }), visibilityTimeoutMs: 30_000, }, + sweepSnapshotOrphans: { + schema: z.object({ + timestamp: z.number(), + lastTimestamp: z.number().optional(), + cron: z.string(), + }), + // The default budget plus two hours, so it stays strictly above the runner's lock TTL + // (budget plus one hour). Ordered that way, a lock outlives the delivery it belongs to. + visibilityTimeoutMs: 18_000_000, + cron: "0 */6 * * *", + jitterInMs: 60_000, + // Load-bearing. A throw takes the dead-letter path, which also reschedules, so the cron chain + // survives a failed pass. With retries it would not behave that way. + retry: { maxAttempts: 1 }, + }, }; diff --git a/internal-packages/run-store/package.json b/internal-packages/run-store/package.json index 7263a6de05c..13075b9c4a9 100644 --- a/internal-packages/run-store/package.json +++ b/internal-packages/run-store/package.json @@ -16,7 +16,8 @@ "dependencies": { "@internal/redis": "workspace:*", "@trigger.dev/core": "workspace:*", - "@trigger.dev/database": "workspace:*" + "@trigger.dev/database": "workspace:*", + "lru-cache": "^11.2.4" }, "devDependencies": { "@internal/run-ops-database": "workspace:*", diff --git a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts index dda0f483bf5..7d88b801a03 100644 --- a/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts +++ b/internal-packages/run-store/src/PostgresRunStore.snapshotWrites.test.ts @@ -8,6 +8,7 @@ import { PostgresRunStore } from "./PostgresRunStore.js"; import { buildCreateRunData, seedSnapshotEnvironment, + seedSnapshotWaitpoints, seedSnapshotWorker, setupSnapshotIdFixture, } from "./testFixtures/snapshotIdFixture.js"; @@ -309,4 +310,94 @@ describe("PostgresRunStore snapshotWrites flag", () => { }) ).rejects.toThrow(/snapshotWrites is off/); }); + + postgresTest( + "a per-org predicate suppresses the snapshot only for the redis-only org", + async ({ prisma }) => { + const ro = await setupSnapshotIdFixture(prisma); + const keep = await setupSnapshotIdFixture(prisma); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: (org) => org !== ro.env.organizationId, + }); + + for (const fx of [ro, keep]) { + await store.completeAttemptSuccess( + fx.run.id, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: fx.env.id, + environmentType: fx.env.type, + projectId: fx.env.projectId, + organizationId: fx.env.organizationId, + }, + }, + { select: { id: true } } + ); + } + + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: ro.run.id } })).toBe(0); + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: keep.run.id } })).toBe( + 1 + ); + } + ); + + postgresTest( + "a per-org predicate suppresses the completed-waitpoint join rows for the redis-only org", + async ({ prisma }) => { + const ro = await setupSnapshotIdFixture(prisma); + const keep = await setupSnapshotIdFixture(prisma); + const store = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + snapshotWrites: (org) => org !== ro.env.organizationId, + }); + + for (const fx of [ro, keep]) { + const { workerId, taskId } = await seedSnapshotWorker(prisma, fx.env); + const waitpointIds = await seedSnapshotWaitpoints(prisma, fx.env, 2); + const snapshotId = generateInternalId(); + await store.lockRunToWorker(fx.run.id, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: snapshotId, + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: fx.env.id, + environmentType: fx.env.type, + projectId: fx.env.projectId, + organizationId: fx.env.organizationId, + completedWaitpointIds: waitpointIds, + completedWaitpointOrder: waitpointIds, + }, + }); + } + + // The redis-only org has no snapshot row, so no join rows link to it either. + expect(await prisma.taskRunExecutionSnapshot.count({ where: { runId: ro.run.id } })).toBe(0); + + const keepSnapshot = await prisma.taskRunExecutionSnapshot.findFirstOrThrow({ + where: { runId: keep.run.id }, + include: { completedWaitpoints: true }, + }); + expect(keepSnapshot.completedWaitpoints).toHaveLength(2); + } + ); }); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 22dc2f90c44..ac4bbf34c92 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -123,9 +123,10 @@ export type PostgresRunStoreOptions = { * When false the store writes no execution-snapshot rows: every nested `executionSnapshots.create` * is omitted and `createExecutionSnapshot` echoes its input instead of inserting. Only the * redis-only dial position sets this, once the Redis store is the sole snapshot writer. - * Defaults to true, so the store behaves exactly as it always has. + * A predicate form decides per run from its organisation id, so a per-org redis-only override + * suppresses only that org's snapshots. Defaults to true, so the store behaves exactly as it always has. */ - snapshotWrites?: boolean; + snapshotWrites?: boolean | ((organizationId?: string) => boolean); }; // A caller sub-select for a relation: `{ select?, include? }` or `true` for a bare `key: true`. @@ -646,7 +647,7 @@ export class PostgresRunStore implements RunStore { private readonly prisma: RunOpsCapableClient; private readonly readOnlyPrisma: RunOpsCapableClient; private readonly schemaVariant: RunStoreSchemaVariant; - private readonly snapshotWrites: boolean; + private readonly snapshotWrites: boolean | ((organizationId?: string) => boolean); private readonly maxWait?: number; private readonly transactionStartRetry?: TransactionStartRetryConfig; @@ -662,6 +663,13 @@ export class PostgresRunStore implements RunStore { this.snapshotWrites = options.snapshotWrites ?? true; } + // Resolves the write flag for one run, from its organisation id when the option is a predicate. + #writesSnapshot(organizationId?: string): boolean { + return typeof this.snapshotWrites === "function" + ? this.snapshotWrites(organizationId) + : this.snapshotWrites; + } + /** * Wraps a nested snapshot create so a single flag removes it everywhere. Prisma treats an absent * key and `undefined` alike, so spreading an empty object drops the nested write entirely rather @@ -674,7 +682,7 @@ export class PostgresRunStore implements RunStore { }; } | Record { - return this.snapshotWrites ? { executionSnapshots: { create } } : {}; + return this.#writesSnapshot(create.organizationId) ? { executionSnapshots: { create } } : {}; } // The writer handle in read-client form, so the routing layer can honor a caller-passed client @@ -1316,7 +1324,7 @@ export class PostgresRunStore implements RunStore { // The join rows link to the snapshot row above. With snapshot writes off there is no such row, // so inserting them would leave dangling links for a snapshot that only the Redis store holds. - if (this.snapshotWrites) { + if (this.#writesSnapshot(data.snapshot.organizationId)) { if (dedicated) { await this.#connectCompletedWaitpoints( prisma, @@ -2036,7 +2044,7 @@ export class PostgresRunStore implements RunStore { // Redis-only: no row is written and the decorator owns the document. Echo the input in the shape // the caller expects, so every caller of this method keeps working while Postgres holds nothing. - if (!this.snapshotWrites) { + if (!this.#writesSnapshot(input.organizationId)) { if (!id) { throw new Error( "PostgresRunStore.createExecutionSnapshot: snapshotWrites is off, so the caller must supply the snapshot id" diff --git a/internal-packages/run-store/src/circuitBreaker.test.ts b/internal-packages/run-store/src/circuitBreaker.test.ts new file mode 100644 index 00000000000..57389028139 --- /dev/null +++ b/internal-packages/run-store/src/circuitBreaker.test.ts @@ -0,0 +1,157 @@ +// The residency cache removes the steady-state round trip, but a process that has not yet seen a run +// still probes once, and under a brownout that one probe costs the full retry budget (measured: about +// 2.07s, from 4 attempts at a 500ms command timeout plus backoff). The breaker bounds that: after a +// few connectivity failures the store stops trying at all, so a sick Redis takes itself off the run +// path without an operator. +import { describe, expect, it } from "vitest"; +import { CircuitBreaker, SnapshotStoreUnavailableError } from "./circuitBreaker.js"; + +const OPTS = { failureThreshold: 3, openDurationMs: 1_000 }; + +function connectivityError(message = "Command timed out"): Error { + return new Error(message); +} + +describe("CircuitBreaker", () => { + it("passes calls through while closed", async () => { + const breaker = new CircuitBreaker(OPTS); + expect(await breaker.run(async () => "ok")).toBe("ok"); + }); + + it("stays closed while failures are below the threshold", async () => { + const breaker = new CircuitBreaker(OPTS); + for (let i = 0; i < 2; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + } + expect(breaker.state).toBe("closed"); + }); + + it("opens on the threshold and then refuses without calling through", async () => { + const breaker = new CircuitBreaker(OPTS); + for (let i = 0; i < 3; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + } + expect(breaker.state).toBe("open"); + + let called = false; + await expect( + breaker.run(async () => { + called = true; + return "ok"; + }) + ).rejects.toBeInstanceOf(SnapshotStoreUnavailableError); + // The point of the breaker: the doomed call is not made, so it costs nothing rather than a + // timeout. + expect(called).toBe(false); + }); + + it("a success resets the count, so intermittent failures never accumulate to an open", async () => { + const breaker = new CircuitBreaker(OPTS); + for (let i = 0; i < 5; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + await breaker.run(async () => "ok"); + } + expect(breaker.state).toBe("closed"); + }); + + it("half-opens after the open window and closes on a successful trial", async () => { + const breaker = new CircuitBreaker({ failureThreshold: 3, openDurationMs: 20 }); + for (let i = 0; i < 3; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + } + expect(breaker.state).toBe("open"); + + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(await breaker.run(async () => "recovered")).toBe("recovered"); + expect(breaker.state).toBe("closed"); + }); + + it("re-opens when the trial call fails again", async () => { + const breaker = new CircuitBreaker({ failureThreshold: 3, openDurationMs: 20 }); + for (let i = 0; i < 3; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + } + await new Promise((resolve) => setTimeout(resolve, 30)); + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + expect(breaker.state).toBe("open"); + }); + + it("ignores a script error, because a bug is not an outage", async () => { + // WRONGTYPE and other Lua errors mean the data or the script is wrong, and every retry against + // every node will fail the same way. Counting them would open the breaker on a defect and take + // the mirror down for runs that are perfectly healthy. + const breaker = new CircuitBreaker(OPTS); + for (let i = 0; i < 10; i++) { + await expect( + breaker.run(async () => + Promise.reject( + new Error("WRONGTYPE Operation against a key holding the wrong kind of value") + ) + ) + ).rejects.toThrow(); + } + expect(breaker.state).toBe("closed"); + }); + it("lets exactly ONE caller through a half-open window", async () => { + // Without this, every concurrent caller reads "half-open" and enters the call, so during an + // outage each one pays the full retry timeout instead of one of them probing recovery. The + // breaker would open again afterwards, but the cost it exists to avoid has already been paid. + const breaker = new CircuitBreaker({ failureThreshold: 3, openDurationMs: 20 }); + for (let i = 0; i < 3; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + } + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(breaker.state).toBe("half-open"); + + let entered = 0; + let release: (() => void) | undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + + // The trial, held pending so the window stays open while the others arrive. + const trial = breaker.run(async () => { + entered += 1; + await held; + return "trial"; + }); + + const others = await Promise.allSettled([ + breaker.run(async () => { + entered += 1; + return "second"; + }), + breaker.run(async () => { + entered += 1; + return "third"; + }), + ]); + + expect(entered).toBe(1); + for (const r of others) { + expect(r.status).toBe("rejected"); + expect((r as PromiseRejectedResult).reason).toBeInstanceOf(SnapshotStoreUnavailableError); + } + + release!(); + await expect(trial).resolves.toBe("trial"); + // The trial succeeded, so the circuit is closed and normal traffic resumes. + expect(breaker.state).toBe("closed"); + }); + + it("frees the half-open slot when the trial fails, rather than wedging shut", async () => { + const breaker = new CircuitBreaker({ failureThreshold: 3, openDurationMs: 20 }); + for (let i = 0; i < 3; i++) { + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + } + await new Promise((resolve) => setTimeout(resolve, 30)); + + await expect(breaker.run(async () => Promise.reject(connectivityError()))).rejects.toThrow(); + expect(breaker.state).toBe("open"); + + // A failed trial re-opens the circuit, and after the next window another single trial is allowed. + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(await breaker.run(async () => "recovered")).toBe("recovered"); + expect(breaker.state).toBe("closed"); + }); +}); diff --git a/internal-packages/run-store/src/circuitBreaker.ts b/internal-packages/run-store/src/circuitBreaker.ts new file mode 100644 index 00000000000..724bc2f2be6 --- /dev/null +++ b/internal-packages/run-store/src/circuitBreaker.ts @@ -0,0 +1,128 @@ +/** + * Thrown instead of attempting a call the breaker has decided will fail. Named so callers can tell + * "Redis is not answering" apart from "Redis answered and said no", which need different handling: + * the first falls back, the second is a real result. + */ +export class SnapshotStoreUnavailableError extends Error { + constructor(message = "snapshot store is unavailable (circuit open)") { + super(message); + this.name = "SnapshotStoreUnavailableError"; + } +} + +export type CircuitBreakerOptions = { + /** Consecutive connectivity failures that open the circuit. */ + failureThreshold?: number; + /** How long the circuit stays open before one trial call is allowed through. */ + openDurationMs?: number; + /** Injectable for tests. Defaults to Date.now. */ + now?: () => number; +}; + +export type CircuitState = "closed" | "open" | "half-open"; + +const DEFAULT_FAILURE_THRESHOLD = 3; +const DEFAULT_OPEN_DURATION_MS = 10_000; + +/** + * Connectivity, not correctness. + * + * A Lua error means the script or the data is wrong: every retry, against every node, fails the same + * way, and no amount of waiting helps. Counting those would open the circuit on a defect and stop + * mirroring runs that are perfectly healthy. A timeout, a closed connection or a down cluster is the + * opposite: nothing about the request is wrong, the server is simply not answering. + */ +function isConnectivityFailure(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + if (/WRONGTYPE|NOSCRIPT|ERR |Lua |user_script/i.test(message)) { + return false; + } + return /timed? ?out|ETIMEDOUT|ECONNREFUSED|ECONNRESET|EPIPE|Connection is closed|Stream isn't writeable|CLUSTERDOWN|max retries|Failed to refresh slots/i.test( + message + ); +} + +/** + * Stops a process paying the retry budget over and over for a Redis that is not answering. + * + * The residency cache already removes the steady-state round trip; this bounds what is left. One + * transition's worth of failed attempts opens the circuit, and every later call returns immediately + * instead of waiting out another timeout. A sick Redis therefore takes ITSELF off the run path, + * which is the property that makes the low dial positions inert under fault without an operator + * setting a second control. + * + * Per process and per client. The sweep holds its own connection and must not trip this one: a long + * scan failing is not evidence that the hot path cannot write. + */ +export class CircuitBreaker { + readonly #failureThreshold: number; + readonly #openDurationMs: number; + readonly #now: () => number; + #consecutiveFailures = 0; + #openedAt?: number; + /** + * Whether a half-open trial is already out. Without it every concurrent caller reads `half-open` + * and enters the call, so during an outage each one pays the full retry timeout, which is the cost + * the breaker exists to avoid. One caller probes recovery; the rest are refused until it settles. + */ + #trialInFlight = false; + + constructor(options: CircuitBreakerOptions = {}) { + this.#failureThreshold = options.failureThreshold ?? DEFAULT_FAILURE_THRESHOLD; + this.#openDurationMs = options.openDurationMs ?? DEFAULT_OPEN_DURATION_MS; + this.#now = options.now ?? Date.now; + } + + get state(): CircuitState { + if (this.#openedAt === undefined) { + return "closed"; + } + return this.#now() - this.#openedAt >= this.#openDurationMs ? "half-open" : "open"; + } + + async run(fn: () => Promise): Promise { + const state = this.state; + + if (state === "open") { + throw new SnapshotStoreUnavailableError(); + } + + // Reserved BEFORE the call, not after: the reservation is the whole point, and checking it + // afterwards would let every arriving caller through first. + if (state === "half-open") { + if (this.#trialInFlight) { + throw new SnapshotStoreUnavailableError(); + } + this.#trialInFlight = true; + } + + try { + const result = await fn(); + this.#consecutiveFailures = 0; + this.#openedAt = undefined; + return result; + } catch (error) { + if (!isConnectivityFailure(error)) { + // A success resets the counter and so does an unrelated error: only an unbroken run of + // connectivity failures is evidence the endpoint is gone. + this.#consecutiveFailures = 0; + throw error; + } + + this.#consecutiveFailures += 1; + // `state` is captured above: reading this.state here would see the value AFTER a re-open and + // could not tell a failed trial from an ordinary failure. + if (state === "half-open" || this.#consecutiveFailures >= this.#failureThreshold) { + this.#openedAt = this.#now(); + this.#consecutiveFailures = 0; + } + throw error; + } finally { + // Always released, whatever the outcome. A trial that threw something unexpected must not + // leave the breaker refusing every caller for the rest of the process's life. + if (state === "half-open") { + this.#trialInFlight = false; + } + } + } +} diff --git a/internal-packages/run-store/src/redisSnapshotStore.gaps.test.ts b/internal-packages/run-store/src/redisSnapshotStore.gaps.test.ts new file mode 100644 index 00000000000..3903e98db85 --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.gaps.test.ts @@ -0,0 +1,301 @@ +// A5. The repair restores the head but not the entries lost in the fork window, so a keyspace ends up +// with a hole in the middle and a correct head. That is invisible at dual-write, where Postgres is +// authoritative and the engine reads the head. It is not invisible at redis-read: the window read +// serves a since-createdAt range straight from Redis, and its guards (a miss, a dangling cycle) +// cannot see a HOLE, so a window that should hold eight entries returns four with nothing logged. A +// history that is short rather than wrong is the harder kind to notice. +// +// Backfilling the lost entries is NOT the fix and would be worse. A late append takes a fresh seq +// from HINCRBY, and both window scripts walk the index in seq order treating it as time order, +// stopping at the first entry past the cursor. A backfilled old entry with a high seq would truncate +// the window harder than the hole does. +// +// So the keyspace records that its history is untrustworthy and windows refuse, which routes the +// caller through its existing miss path to Postgres. Point reads stay Redis-served, because the +// repair does guarantee the head converges. +import { describe, expect } from "vitest"; +import { redisTest } from "@internal/testcontainers"; +import { RedisSnapshotStore, snapshotKeys } from "./redisSnapshotStore.js"; +import type { SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { createRedisClient } from "@internal/redis"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function entry(runId: string, id: string, createdAt: string): SnapshotEntryInput { + return { + id, + engine: "V2", + executionStatus: "EXECUTING", + description: "d", + runId, + runStatus: "EXECUTING", + createdAt, + environmentId: "env_1", + environmentType: "DEVELOPMENT", + projectId: "proj_1", + organizationId: "org_1", + }; +} + +const at = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds)).toISOString(); + +async function seed(store: RedisSnapshotStore, runId: string, count: number): Promise { + for (let n = 0; n < count; n++) { + await store.append({ + entry: entry(runId, `snap_${n}`, at(n)), + kind: n === 0 ? "birth" : "transition", + isTerminal: false, + }); + } +} + +describe("the gaps marker", () => { + redisTest( + "is unset on a healthy keyspace, which still serves its window", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_healthy"; + await seed(store, runId, 5); + + expect(await store.hasGaps(runId)).toBe(false); + expect((await store.getSinceCreatedAt(runId, at(1))).kind).toBe("hit"); + expect((await store.getSince(runId, "snap_1")).kind).toBe("hit"); + } finally { + await store.quit(); + } + } + ); + + redisTest("makes BOTH window reads refuse, so each falls back", async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_holed"; + await seed(store, runId, 5); + + // What a repair does when it lands: the head is right, the window is not to be trusted. + await store.markGaps(runId); + expect(await store.hasGaps(runId)).toBe(true); + + // Both window commands, because a caller that fell back on one and not the other would still + // serve a short history through the second. + expect((await store.getSinceCreatedAt(runId, at(1))).kind).toBe("miss"); + expect((await store.getSince(runId, "snap_1")).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); + + redisTest( + "leaves point reads alone, because the repair converges the head", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_point"; + await seed(store, runId, 5); + await store.markGaps(runId); + + // The head is the engine's hot read and the repair guarantees it. Refusing it would send every + // transition of a once-forked run to Postgres for the rest of its life. + const head = await store.getLatest(runId); + expect(head?.entry.id).toBe("snap_4"); + + const byId = await store.getById(runId, "snap_2"); + expect(byId?.entry.id).toBe("snap_2"); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "is set by a fork, which is direct evidence of divergence", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_forked"; + await seed(store, runId, 3); + + const result = await store.append({ + entry: entry(runId, "snap_late", at(9)), + kind: "transition", + isTerminal: false, + expectedCur: "snap_wrong", + }); + + expect(result.outcome).toBe("forked"); + // A fork means this keyspace and Postgres already disagree about the head, so whatever the + // repair does later, the window between them is not trustworthy now. + expect(await store.hasGaps(runId)).toBe(true); + expect((await store.getSinceCreatedAt(runId, at(1))).kind).toBe("miss"); + } finally { + await store.quit(); + } + } + ); + + redisTest( + "dies with the keyspace rather than needing its own expiry", + async ({ redisOptions }) => { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const runId = "run_ttl"; + await seed(store, runId, 3); + await store.markGaps(runId); + + // The marker is a field on the seq hash, so the completion expiry that governs the keyspace + // governs it too. No second lifetime to get wrong. + expect(await probe.hget(snapshotKeys(runId).seq, "g")).toBe("1"); + await store.dropRun(runId); + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + } finally { + await Promise.all([store.quit(), probe.quit().catch(() => {})]); + } + } + ); + redisTest( + "a lone seq key is never created for a run that has no keyspace", + async ({ redisOptions }) => { + // markGaps writes a field on the seq hash, and HSET creates the hash if it is absent. For a + // run with no keyspace that would leave a stray seq key holding only the marker: keyspaceAlive + // stays false so no read is affected, but the sweeper scans on the entry hash and would never + // discover it, so it would never be reaped either. An unbounded leak with no reader. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const runId = "run_never_born"; + expect(await store.markGapsIfResident(runId)).toBe(false); + expect(await probe.exists(snapshotKeys(runId).seq)).toBe(0); + + await seed(store, runId, 2); + expect(await store.markGapsIfResident(runId)).toBe(true); + expect(await store.hasGaps(runId)).toBe(true); + } finally { + await Promise.all([store.quit(), probe.quit().catch(() => {})]); + } + } + ); + redisTest( + "a transition that finds the index gone marks the history, rather than rebuilding a partial one", + async ({ redisOptions }) => { + // keyspaceAlive tests the entry hash and seq, not the index. An index lost while those two + // survive used to let the next transition recreate it holding only that entry, and a window + // read would then see a live index, report a hit, and return one entry as the whole range. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const runId = "run_index_lost"; + await seed(store, runId, 4); + expect(await store.hasGaps(runId)).toBe(false); + + // Lose the index only, the way a per-key expiry or eviction would. + await probe.del(snapshotKeys(runId).idx); + expect(await probe.exists(snapshotKeys(runId).e)).toBe(1); + + await store.append({ + entry: entry(runId, "snap_after_loss", at(9)), + kind: "transition", + isTerminal: false, + }); + + // The index is back, holding one entry, which is exactly the trap. + expect(await probe.exists(snapshotKeys(runId).idx)).toBe(1); + // So the keyspace is marked and the window refuses instead of serving a one-entry range. + expect(await store.hasGaps(runId)).toBe(true); + expect((await store.getSinceCreatedAt(runId, at(1))).kind).toBe("miss"); + + // The head still moves: refusing the transition would have frozen it. + expect((await store.getLatest(runId))?.entry.id).toBe("snap_after_loss"); + } finally { + await Promise.all([store.quit(), probe.quit().catch(() => {})]); + } + } + ); + + redisTest( + "dropping a run removes its wait cycle keys even when seq is already gone", + async ({ redisOptions }) => { + // The cycle count lives on seq, so seq being absent read as zero cycles and left every wait + // cycle key behind, while dropRun claimed to remove the whole keyspace. The sweep cannot see + // those either: it discovers keyspaces by the entry hash. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const runId = "run_orphan_cycles"; + // Derived, never hardcoded: snapshotKeys returns UNPREFIXED keys and the `engine:` prefix is + // the client's, which testcontainers do not set. A literal prefix here creates keys the + // script never looks at, and the test passes or fails for the wrong reason. + const eKey = snapshotKeys(runId).e; + const base = eKey.slice(0, -2); + await seed(store, runId, 2); + + // Wait cycle keys as the append script writes them, then lose seq. + await probe.hset(`${base}:wp:1`, "order", "[]", "count", "0", "distinct", "[]"); + await probe.hset(`${base}:wp:2`, "order", "[]", "count", "0", "distinct", "[]"); + await probe.del(snapshotKeys(runId).seq); + + await store.dropRun(runId); + + for (const key of [`${base}:wp:1`, `${base}:wp:2`]) { + expect(await probe.exists(key)).toBe(0); + } + for (const key of Object.values(snapshotKeys(runId))) { + expect(await probe.exists(key)).toBe(0); + } + } finally { + await Promise.all([store.quit(), probe.quit().catch(() => {})]); + } + } + ); + redisTest("drops a SPARSE wait cycle key with seq already gone", async ({ redisOptions }) => { + // A miss-streak early exit was wrong. Cycle keys can be sparse, so with seq absent (count reads + // as zero) and only wp:10 alive, stopping after a run of absent keys left it behind, and the + // entry hash is deleted in the same call so the sweep could never discover it either. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const runId = "run_sparse_cycles"; + const base = snapshotKeys(runId).e.slice(0, -2); + await seed(store, runId, 2); + + await probe.hset(`${base}:wp:10`, "order", "[]"); + await probe.del(snapshotKeys(runId).seq); + + await store.dropRun(runId); + + expect(await probe.exists(`${base}:wp:10`)).toBe(0); + for (const key of Object.values(snapshotKeys(runId))) { + expect(await probe.exists(key)).toBe(0); + } + } finally { + await Promise.all([store.quit(), probe.quit().catch(() => {})]); + } + }); + + redisTest("marks gaps when the repair's append is a DUPLICATE", async ({ redisOptions }) => { + // The script returned on duplicate before reaching the marker, so a repair whose entry had + // already landed left the keyspace serving short windows as though they were whole. A repair + // runs BECAUSE an append was lost, so the entries either side are gone regardless. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + try { + const runId = "run_dup_gaps"; + await seed(store, runId, 3); + expect(await store.hasGaps(runId)).toBe(false); + + // Re-append an id that is already present but is not the head. + const result = await store.append({ + entry: entry(runId, "snap_1", at(1)), + kind: "transition", + isTerminal: false, + markGaps: true, + }); + + expect(result.outcome).toBe("duplicate"); + expect(await store.hasGaps(runId)).toBe(true); + expect((await store.getSinceCreatedAt(runId, at(0))).kind).toBe("miss"); + } finally { + await store.quit(); + } + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.outcomes.test.ts b/internal-packages/run-store/src/redisSnapshotStore.outcomes.test.ts new file mode 100644 index 00000000000..839240ebe73 --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.outcomes.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { APPEND_RESULT_OUTCOMES } from "./redisSnapshotStore.js"; +import type { AppendResult } from "./redisSnapshotStore.js"; + +describe("append result outcomes", () => { + it("lists every outcome the store can return", () => { + // A type error here means AppendResult moved and the list did not follow. The metrics layer + // bounds against this list, so an omitted outcome collapses to "other". + type Declared = (typeof APPEND_RESULT_OUTCOMES)[number]; + type Actual = AppendResult["outcome"]; + type AssertSame = [A] extends [B] ? ([B] extends [A] ? true : never) : never; + const _covers: AssertSame = true; + void _covers; + + expect([...APPEND_RESULT_OUTCOMES].sort()).toEqual([ + "duplicate", + "forked", + "skippedNoKeyspace", + "written", + ]); + }); + + it("uses the result vocabulary, not the Lua wire vocabulary", () => { + expect(APPEND_RESULT_OUTCOMES).not.toContain("skipped"); + expect(APPEND_RESULT_OUTCOMES).toContain("skippedNoKeyspace"); + }); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.readLiveness.test.ts b/internal-packages/run-store/src/redisSnapshotStore.readLiveness.test.ts new file mode 100644 index 00000000000..34a927fbb2d --- /dev/null +++ b/internal-packages/run-store/src/redisSnapshotStore.readLiveness.test.ts @@ -0,0 +1,166 @@ +// The write guard and the read path must agree on what a live keyspace is. The append script +// refuses a transition unless BOTH `e` and `seq` exist; a read that keys off `cur` or `e` alone +// keeps serving the frozen head after `seq` is evicted, and a stale hit is not a miss, so the +// decorator's Postgres fallback never fires and the two stores diverge permanently. +import { createRedisClient, type RedisOptions } from "@internal/redis"; +import { redisTest } from "@internal/testcontainers"; +import { describe, expect } from "vitest"; +import { RedisSnapshotStore, snapshotKeys, type SnapshotEntryInput } from "./redisSnapshotStore.js"; + +function entry(over: Partial = {}): SnapshotEntryInput { + return { + id: "snap_1", + engine: "V2", + executionStatus: "RUN_CREATED", + description: "created", + runId: "run_1", + runStatus: "PENDING", + createdAt: "2026-08-21T00:00:00.000Z", + environmentId: "env_1", + environmentType: "PRODUCTION", + projectId: "proj_1", + organizationId: "org_1", + ...over, + }; +} + +async function seededStore(redisOptions: RedisOptions) { + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + await store.append({ entry: entry({ id: "snap_1" }), kind: "birth", isTerminal: false }); + await store.append({ + entry: entry({ id: "snap_2", createdAt: "2026-08-21T00:00:01.000Z" }), + kind: "transition", + isTerminal: false, + cycle: { kind: "new", completedWaitpoints: [{ id: "w_a", index: 0 }] }, + }); + return store; +} + +async function evict(redisOptions: RedisOptions, key: string) { + const raw = createRedisClient(redisOptions); + try { + await raw.del(key); + } finally { + await raw.quit(); + } +} + +describe("read liveness anchors", () => { + redisTest("getLatest misses once the seq anchor is gone", async ({ redisOptions }) => { + const store = await seededStore(redisOptions); + try { + expect(await store.getLatest("run_1")).not.toBeNull(); + await evict(redisOptions, snapshotKeys("run_1").seq); + expect(await store.getLatest("run_1")).toBeNull(); + } finally { + await store.quit(); + } + }); + + redisTest("getById misses once the seq anchor is gone", async ({ redisOptions }) => { + const store = await seededStore(redisOptions); + try { + expect(await store.getById("run_1", "snap_2")).not.toBeNull(); + await evict(redisOptions, snapshotKeys("run_1").seq); + expect(await store.getById("run_1", "snap_2")).toBeNull(); + } finally { + await store.quit(); + } + }); + + redisTest( + "getSnapshotWaitpointIds reports not present once the seq anchor is gone", + async ({ redisOptions }) => { + const store = await seededStore(redisOptions); + try { + expect(await store.getSnapshotWaitpointIds("run_1", "snap_2")).toMatchObject({ + present: true, + }); + await evict(redisOptions, snapshotKeys("run_1").seq); + expect(await store.getSnapshotWaitpointIds("run_1", "snap_2")).toEqual({ + present: false, + distinctIds: [], + order: [], + }); + } finally { + await store.quit(); + } + } + ); + + redisTest("getSinceCreatedAt misses once the seq anchor is gone", async ({ redisOptions }) => { + const store = await seededStore(redisOptions); + try { + expect(await store.getSinceCreatedAt("run_1", "2026-08-21T00:00:00.000Z")).toMatchObject({ + kind: "hit", + }); + await evict(redisOptions, snapshotKeys("run_1").seq); + expect(await store.getSinceCreatedAt("run_1", "2026-08-21T00:00:00.000Z")).toEqual({ + kind: "miss", + }); + } finally { + await store.quit(); + } + }); + + redisTest("getSince misses once the seq anchor is gone", async ({ redisOptions }) => { + const store = await seededStore(redisOptions); + try { + expect(await store.getSince("run_1", "snap_1")).toMatchObject({ kind: "hit" }); + await evict(redisOptions, snapshotKeys("run_1").seq); + expect(await store.getSince("run_1", "snap_1")).toEqual({ kind: "miss" }); + } finally { + await store.quit(); + } + }); + + redisTest("getSince misses once the index anchor is gone", async ({ redisOptions }) => { + const store = await seededStore(redisOptions); + try { + await evict(redisOptions, snapshotKeys("run_1").idx); + // The sibling window command (getSinceCreatedAt) already refuses a lost index. Serving an + // empty HIT here would report "nothing new" for the rest of the run's life. + expect(await store.getSince("run_1", "snap_1")).toEqual({ kind: "miss" }); + } finally { + await store.quit(); + } + }); + + // The property the whole change exists for: whatever the write guard refuses, no read serves. + redisTest( + "a keyspace that refuses a transition serves no read either", + async ({ redisOptions }) => { + for (const anchor of ["e", "seq"] as const) { + const runId = `run_coherence_${anchor}`; + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: 60_000 }); + try { + await store.append({ + entry: entry({ id: "snap_1", runId }), + kind: "birth", + isTerminal: false, + }); + await evict(redisOptions, snapshotKeys(runId)[anchor]); + + const write = await store.append({ + entry: entry({ id: "snap_2", runId }), + kind: "transition", + isTerminal: false, + }); + expect(write).toEqual({ outcome: "skippedNoKeyspace" }); + + expect(await store.getLatest(runId)).toBeNull(); + expect(await store.getById(runId, "snap_1")).toBeNull(); + expect(await store.getSnapshotWaitpointIds(runId, "snap_1")).toMatchObject({ + present: false, + }); + expect(await store.getSince(runId, "snap_1")).toEqual({ kind: "miss" }); + expect(await store.getSinceCreatedAt(runId, "2026-08-20T00:00:00.000Z")).toEqual({ + kind: "miss", + }); + } finally { + await store.quit(); + } + } + } + ); +}); diff --git a/internal-packages/run-store/src/redisSnapshotStore.ts b/internal-packages/run-store/src/redisSnapshotStore.ts index 2339ab0dd5e..45e98ad9073 100644 --- a/internal-packages/run-store/src/redisSnapshotStore.ts +++ b/internal-packages/run-store/src/redisSnapshotStore.ts @@ -7,6 +7,8 @@ import { } from "@internal/redis"; import { Logger } from "@trigger.dev/core/logger"; import type { CompletedWaitpoint } from "@trigger.dev/core/v3/schemas"; +import { CircuitBreaker, type CircuitBreakerOptions } from "./circuitBreaker.js"; +import { ResidencyCache } from "./residencyCache.js"; export type SnapshotKeys = { e: string; idx: string; cur: string; seq: string }; @@ -195,8 +197,27 @@ export type AppendResult = | { outcome: "forked"; actualCur: string } | { outcome: "duplicate"; seq: number }; +/** The single source of truth for the outcome vocabulary the metrics layer bounds against. */ +export const APPEND_RESULT_OUTCOMES = [ + "written", + "skippedNoKeyspace", + "forked", + "duplicate", +] as const satisfies readonly AppendResult["outcome"][]; + +/** + * `satisfies` alone only proves each listed literal is a valid outcome. This proves the reverse too, + * so a new member on AppendResult fails the build here rather than becoming "other" on a dashboard. + */ +type AssertSameOutcomes = [A] extends [B] ? ([B] extends [A] ? true : never) : never; +const _outcomesExhaustive: AssertSameOutcomes< + (typeof APPEND_RESULT_OUTCOMES)[number], + AppendResult["outcome"] +> = true; +void _outcomesExhaustive; + export type SnapshotStoreMetrics = { - recordAppend(outcome: string, ttl: string): void; + recordAppend(outcome: string, ttl: string, organizationId?: string): void; recordEntryBytes(bytes: number): void; recordCycleKeyBytes(bytes: number): void; recordCycleCount(count: number): void; @@ -220,6 +241,10 @@ export type RedisSnapshotStoreConnection = export type RedisSnapshotStoreOptions = RedisSnapshotStoreConnection & { completedTtlMs: number; + /** Entries in the per-process residency cache. See {@link ResidencyCache}. */ + residencyCacheMax?: number; + /** Tuning for the per-process breaker. See {@link CircuitBreaker}. */ + breaker?: CircuitBreakerOptions; sinceLimit?: number; highWater?: { entryBytes?: number; cycleKeyBytes?: number; cycleCount?: number }; metrics?: SnapshotStoreMetrics; @@ -250,6 +275,19 @@ export class RedisSnapshotStore { private readonly sinceLimit: number; private readonly metrics?: SnapshotStoreMetrics; private readonly highWater: NonNullable; + /** + * Which runs this process knows the mirror does not own, so a transition for one of them never + * reaches the network. Lives here rather than on the decorator because the decorator is re-minted + * per transaction while this store instance is shared, and because this class owns the append + * replies that are the cache's only ground truth. + */ + readonly #residency: ResidencyCache; + /** + * Bounds what the residency cache cannot: the first probe for a run this process has not seen, + * which under a brownout costs the whole retry budget. After a few connectivity failures the store + * stops calling out at all, so a sick Redis removes itself from the run path with no operator. + */ + readonly #breaker: CircuitBreaker; #quit?: Promise; constructor(options: RedisSnapshotStoreOptions) { @@ -258,6 +296,10 @@ export class RedisSnapshotStore { this.sinceLimit = options.sinceLimit ?? 50; this.metrics = options.metrics; this.highWater = options.highWater ?? {}; + this.#residency = new ResidencyCache({ + ...(options.residencyCacheMax !== undefined && { max: options.residencyCacheMax }), + }); + this.#breaker = new CircuitBreaker(options.breaker ?? {}); this.ownsClient = options.client === undefined; this.redis = options.client ?? @@ -283,20 +325,81 @@ export class RedisSnapshotStore { await this.#quit; } + /** + * Every command goes through here, so the breaker sits on the one seam rather than on each method. + * Latency is still recorded for a refused call: a call that cost nothing because the circuit was + * open is exactly the thing an operator wants to see in the latency series. + */ async #timed(op: string, fn: () => Promise): Promise { const started = Date.now(); try { - return await fn(); + return await this.#breaker.run(fn); } finally { this.metrics?.recordLatency(op, Date.now() - started); } } + /** Test seam. */ + get breakerState(): "closed" | "open" | "half-open" { + return this.#breaker.state; + } + + /** + * Records that this run's Redis history has a hole, so window reads must not serve it. Separate + * from the append path because a repair can conclude the head is already current and still know + * that entries were lost. + */ + async markGaps(runId: string): Promise { + await this.redis.hset(snapshotKeys(runId).seq, "g", "1"); + } + + /** + * Marks only a keyspace that exists, and reports whether it did. + * + * The unconditional form must not be used on a run whose residency is unknown: HSET creates the + * hash, so a non-resident run would be left holding a lone `seq` key with nothing but the marker. + * `keyspaceAlive` would stay false so no read would be affected, but the sweeper discovers + * keyspaces by scanning for the ENTRY hash, so it would never find that key either. An unbounded + * leak with no reader is the one outcome worse than the hole this marker exists to report. + */ + async markGapsIfResident(runId: string): Promise { + const k = snapshotKeys(runId); + return this.#timed("markGapsIfResident", async () => { + const marked = await this.redis.markSnapshotGaps(k.e, k.seq); + return marked === 1; + }); + } + + async hasGaps(runId: string): Promise { + return (await this.redis.hget(snapshotKeys(runId).seq, "g")) === "1"; + } + + /** Test seam. */ + residencyFor(runId: string): "resident" | "non-resident" | undefined { + return this.#residency.get(runId); + } + + /** + * Removes a run's whole keyspace, wait-cycle keys included. The caller must have established that + * the head cannot be trusted and that Postgres still holds the run's rows. + */ + async dropRun(runId: string): Promise { + const keys = snapshotKeys(runId); + await this.redis.dropSnapshotRun(keys.e, keys.idx, keys.cur, keys.seq); + // This process, at least, stops asking. Other processes learn it from their next append. + this.#residency.setNonResident(runId); + } + async append(args: { entry: SnapshotEntryInput; kind: "birth" | "transition"; isTerminal: boolean; expectedCur?: string; + /** + * Marks the keyspace as holed, so window reads refuse and fall back to Postgres. Set by the + * repair, which only runs because an append was lost. + */ + markGaps?: boolean; cycle?: | { kind: "new"; @@ -324,6 +427,15 @@ export class RedisSnapshotStore { "Writing it into the entry JSON breaks byte-comparability with the Postgres row." ); } + // The whole point of the cache. A transition into a keyspace this process already knows is gone + // is refused without a round trip. A birth deliberately never takes this path: a birth is what + // CREATES residency, so it must reach the script even when the run is currently unknown. + if (args.kind === "transition" && this.#residency.get(args.entry.runId) === "non-resident") { + this.metrics?.recordSkippedNoKeyspace(); + this.metrics?.recordAppend("skippedNoKeyspace", "none", args.entry.organizationId); + return { outcome: "skippedNoKeyspace" }; + } + return this.#timed("append", async () => { const k = snapshotKeys(args.entry.runId); const raw = JSON.stringify(args.entry); @@ -375,10 +487,18 @@ export class RedisSnapshotStore { orderCount, args.expectedCur ?? "", args.expectedCur !== undefined ? "1" : "0", - distinctJson + distinctJson, + args.markGaps ? "1" : "0" )) as string[]; - return this.#interpretAppend(reply, raw, orderJson, records, args.entry.runId); + return this.#interpretAppend( + reply, + raw, + orderJson, + records, + args.entry.runId, + args.entry.organizationId + ); }); } @@ -387,19 +507,27 @@ export class RedisSnapshotStore { raw: string, orderJson: string, records: string, - runId: string + runId: string, + organizationId: string ): AppendResult { if (reply[0] === SKIPPED) { + // Authoritative and final: the script looked and there is no keyspace. Only a birth could + // create one and this run's birth has already happened. + this.#residency.setNonResident(runId); this.metrics?.recordSkippedNoKeyspace(); - this.metrics?.recordAppend("skippedNoKeyspace", "none"); + this.metrics?.recordAppend("skippedNoKeyspace", "none", organizationId); return { outcome: "skippedNoKeyspace" }; } if (reply[0] === FORKED) { - this.metrics?.recordAppend("forked", "none"); + // A fork means the script found a keyspace and disagreed about its head, so the run IS + // resident. + this.#residency.setResident(runId); + this.metrics?.recordAppend("forked", "none", organizationId); return { outcome: "forked", actualCur: reply[1] ?? "" }; } if (reply[0] === DUPLICATE) { - this.metrics?.recordAppend("duplicate", "none"); + this.#residency.setResident(runId); + this.metrics?.recordAppend("duplicate", "none", organizationId); return { outcome: "duplicate", seq: Number(reply[1]) }; } const seq = Number(reply[1]); @@ -409,8 +537,10 @@ export class RedisSnapshotStore { if (cycleMismatch) { this.metrics?.recordCycleMismatch(); } + // A written entry proves the keyspace exists. For a birth this is what makes the run resident. + this.#residency.setResident(runId); this.#observeSizes(raw, orderJson, records, cycleSeq, runId); - this.metrics?.recordAppend("written", ttl); + this.metrics?.recordAppend("written", ttl, organizationId); return { outcome: "written", seq, @@ -701,6 +831,12 @@ export class RedisSnapshotStore { local eKey, idxKey, curKey, seqKey = KEYS[1], KEYS[2], KEYS[3], KEYS[4] local base = string.sub(eKey, 1, #eKey - 2) local function wpKey(n) return base .. ':wp:' .. n end + -- The ONE liveness test, shared by the write guard and every read. Two anchors, because keys + -- expire independently and eviction takes whole keys: seq can be gone while e and cur + -- survive, and a read answering from cur there serves a frozen head no write can advance. + local function keyspaceAlive() + return redis.call('EXISTS', eKey) == 1 and redis.call('EXISTS', seqKey) == 1 + end local function orderFor(pointer) if not pointer then return '' end local cs = string.match(pointer, '^(%d+):') @@ -727,6 +863,21 @@ export class RedisSnapshotStore { end `; + this.redis.defineCommand("markSnapshotGaps", { + numberOfKeys: 2, + lua: ` + local eKey = KEYS[1] + local seqKey = KEYS[2] + -- Both anchors, the same pair keyspaceAlive uses. Marking on the strength of one of them + -- would create the other. + if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', seqKey) == 0 then + return 0 + end + redis.call('HSET', seqKey, 'g', '1') + return 1 + `, + }); + this.redis.defineCommand("appendSnapshotEntry", { numberOfKeys: 4, lua: ` @@ -747,12 +898,14 @@ export class RedisSnapshotStore { -- The COMPLETE distinct id set. Not the order deduped: order omits every id with no batch -- index, and those ids still have to come back on a read. local distinctJson = ARGV[14] - - -- Liveness is TWO anchors: e and seq. All keys get the same PEXPIRE but expire independently - -- (or seq can vanish under maxmemory eviction while e survives), so checking e alone lets a - -- late transition recreate seq with no TTL and restart it at 1 beside a surviving idx. A - -- birth always creates both in this same script, so this never rejects a live keyspace. - if kind == 'transition' and (redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', seqKey) == 0) then + -- Set by the repair. A repair exists BECAUSE an append was lost, so whatever it manages to + -- put back, the entries between are gone and the window is short. + local markGaps = ARGV[15] == '1' + + -- Checking e alone would let a late transition recreate seq with no TTL and restart it at 1 + -- beside a surviving idx. A birth always creates both in this same script, so this never + -- rejects a live keyspace. + if kind == 'transition' and not keyspaceAlive() then return { '${SKIPPED}' } end @@ -760,6 +913,13 @@ export class RedisSnapshotStore { -- CAS below -- a present id can only be this same retry, never a competitor's write. local prior = redis.call('HGET', eKey, id .. '#s') if prior then + -- Marked before returning. The caller that asks for a mark is the repair, and a repair + -- runs BECAUSE an append was lost, so the entries either side are gone whether or not + -- this particular id had already landed. Returning early without marking left the + -- keyspace serving short windows as though they were whole. + if markGaps then + redis.call('HSET', seqKey, 'g', '1') + end return { '${DUPLICATE}', prior } end @@ -769,10 +929,29 @@ export class RedisSnapshotStore { if casEnabled then local actual = redis.call('GET', curKey) if (actual or '') ~= expectedCur then + -- The one mutation a refused append makes, and it is not part of the append. A fork means + -- this keyspace and Postgres already disagree about the head, so its history cannot be + -- served as a window until something re-establishes that it can. The entry itself is + -- still not written. + redis.call('HSET', seqKey, 'g', '1') return { '${FORKED}', actual or '' } end end + + + -- The index can go while the entry hash and seq survive, and keyspaceAlive does not test it. + -- This append is about to recreate it holding only the new entry, and a window read would + -- then see a live index, report a HIT, and return that one entry as though it were the whole + -- range. Same silent short history as a lost append, so it is recorded the same way: the head + -- keeps moving and window reads fall back to Postgres, which still holds the log. + -- + -- Refusing the transition instead would freeze the head, which is the outcome this whole area + -- exists to avoid. + if kind == 'transition' and redis.call('EXISTS', idxKey) == 0 then + redis.call('HSET', seqKey, 'g', '1') + end + local seq = redis.call('HINCRBY', seqKey, 'e', 1) local cycleSeq = 0 @@ -859,10 +1038,38 @@ export class RedisSnapshotStore { `, }); + this.redis.defineCommand("dropSnapshotRun", { + numberOfKeys: 4, + lua: ` + ${PRELUDE} + -- The high-water mark, when seq still has it. It is the fast path and the common one. + local cycles = tonumber(redis.call('HGET', seqKey, 'c') or '0') + for i = 1, cycles do + redis.call('DEL', wpKey(i)) + end + + -- seq holds the count, so seq being gone used to mean the count read as 0 and every wait + -- cycle key was left behind, while this command claimed to remove the whole keyspace. An + -- orphan the sweep cannot see either, because it discovers keyspaces by the entry hash. + -- + -- So sweep a bounded range unconditionally. A miss-streak early exit was wrong: cycle keys + -- can be SPARSE, so with seq gone and only wp:10 alive, stopping after a run of absent keys + -- leaves it behind, and the entry hash is deleted below so the sweep can never find it + -- either. Every key here shares the {runId} tag, so this stays inside one slot, and the + -- bound keeps a pathological run from turning a drop into a long script. + for probe = cycles + 1, cycles + 512 do + redis.call('DEL', wpKey(probe)) + end + + return redis.call('DEL', eKey, idxKey, curKey, seqKey) + `, + }); + this.redis.defineCommand("readSnapshotById", { numberOfKeys: 4, lua: ` ${PRELUDE} + if not keyspaceAlive() then return nil end local id = ARGV[1] local vals = redis.call('HMGET', eKey, id, id .. '#s', id .. '#c') if not vals[1] then return nil end @@ -875,6 +1082,7 @@ export class RedisSnapshotStore { numberOfKeys: 4, lua: ` ${PRELUDE} + if not keyspaceAlive() then return nil end local cur = redis.call('GET', curKey) if not cur then return nil end local vals = redis.call('HMGET', eKey, cur, cur .. '#s', cur .. '#c') @@ -888,6 +1096,9 @@ export class RedisSnapshotStore { lua: ` ${PRELUDE} local id = ARGV[1] + -- Not present, which is what sends the caller to Postgres. An empty id set from an + -- incomplete keyspace would read as authoritative. + if not keyspaceAlive() then return { '0', '' } end if redis.call('HEXISTS', eKey, id) == 0 then return { '0', '' } end @@ -909,7 +1120,12 @@ export class RedisSnapshotStore { -- Both anchors, for the reason the append script gives: keys expire independently, and an -- index lost to eviction while the entry hash survives would otherwise report an empty HIT -- on every poll for the rest of the run's life, with Postgres holding the transitions. - if redis.call('EXISTS', eKey) == 0 or redis.call('EXISTS', idxKey) == 0 then return nil end + if not keyspaceAlive() or redis.call('EXISTS', idxKey) == 0 then return nil end + + -- A keyspace that lost an append has a hole, and no guard downstream can see one: a window + -- that should hold eight entries would return four and look complete. Refuse, and the + -- caller's existing miss path asks Postgres, which still holds the whole log. + if redis.call('HGET', seqKey, 'g') == '1' then return nil end -- STRICTLY greater than the cursor, and same-millisecond entries are dropped. Postgres -- serves this window with createdAt > cursor and drops them too; a Redis read that is more @@ -974,6 +1190,14 @@ export class RedisSnapshotStore { local sinceId = ARGV[1] local limit = tonumber(ARGV[2]) + -- Same gate as the sibling window command: without it a lost index reports an empty HIT, + -- so the caller stops asking Postgres for a window Postgres alone still holds. + if not keyspaceAlive() or redis.call('EXISTS', idxKey) == 0 then return nil end + + -- And the same hole gate, for the same reason. A caller that fell back on one window command + -- and not the other would still serve a short history through the second. + if redis.call('HGET', seqKey, 'g') == '1' then return nil end + -- The index holds valid entries only, so an invalid since id misses ZSCORE. Its seq is still -- on its own '#s' field, which keeps the id resolvable without indexing invalid rows. local score = redis.call('ZSCORE', idxKey, sinceId) @@ -1046,6 +1270,18 @@ export function decodeWaitpointIds( declare module "@internal/redis" { interface RedisCommander { + dropSnapshotRun( + eKey: string, + idxKey: string, + curKey: string, + seqKey: string, + callback?: Callback + ): Result; + markSnapshotGaps( + eKey: string, + seqKey: string, + callback?: Callback + ): Result; appendSnapshotEntry( eKey: string, idxKey: string, @@ -1065,6 +1301,7 @@ declare module "@internal/redis" { expectedCur: string, casEnabled: string, distinctJson: string, + markGaps: string, callback?: Callback ): Result; readSnapshotById( diff --git a/internal-packages/run-store/src/residencyCache.test.ts b/internal-packages/run-store/src/residencyCache.test.ts new file mode 100644 index 00000000000..3f280d65710 --- /dev/null +++ b/internal-packages/run-store/src/residencyCache.test.ts @@ -0,0 +1,59 @@ +// The keyspace is a run's residency record, and it lives in Redis, so the store had to ask Redis to +// learn a run was NOT its own. That put Redis on the hot path for every transition of every run, +// resident or not: 2 percent with a healthy Redis, 4 times with a slow one, and it never decayed, +// because a fleet with no resident runs still asked once per transition. +// +// The cache is sound because residency is monotonic. Only a birth creates a keyspace: the append +// script refuses `kind: "transition"` into a dead one, and the repair appends as a transition. So +// non-resident is permanent, and a cached negative can never suppress a resident run's append. +import { describe, expect, it } from "vitest"; +import { ResidencyCache } from "./residencyCache.js"; + +describe("ResidencyCache", () => { + it("has no opinion about a run it has never seen", () => { + expect(new ResidencyCache().get("run_a")).toBeUndefined(); + }); + + it("remembers a run that has no keyspace, which is the whole point", () => { + const cache = new ResidencyCache(); + cache.setNonResident("run_a"); + expect(cache.get("run_a")).toBe("non-resident"); + }); + + it("remembers a run that has one", () => { + const cache = new ResidencyCache(); + cache.setResident("run_a"); + expect(cache.get("run_a")).toBe("resident"); + }); + + it("never lets a negative be overwritten by a positive", () => { + // Non-resident is permanent, so a positive arriving afterwards is stale information, not news. + // Honouring it would start mirroring a run half way through its life, which is the one thing + // the residency model forbids. + const cache = new ResidencyCache(); + cache.setNonResident("run_a"); + cache.setResident("run_a"); + expect(cache.get("run_a")).toBe("non-resident"); + }); + + it("lets a positive be replaced by a negative", () => { + // The safe direction: a keyspace can go away under a completion expiry, a sweep, or an + // eviction. Believing that costs nothing, because the append script refuses the write anyway. + const cache = new ResidencyCache(); + cache.setResident("run_a"); + cache.setNonResident("run_a"); + expect(cache.get("run_a")).toBe("non-resident"); + }); + + it("stays inside its bound", () => { + const cache = new ResidencyCache({ max: 10 }); + for (let i = 0; i < 100; i++) { + cache.setNonResident(`run_${i}`); + } + expect(cache.size).toBeLessThanOrEqual(10); + // An evicted entry is a cache miss, never a wrong answer: the next transition probes Redis once + // and re-learns it. + expect(cache.get("run_0")).toBeUndefined(); + expect(cache.get("run_99")).toBe("non-resident"); + }); +}); diff --git a/internal-packages/run-store/src/residencyCache.ts b/internal-packages/run-store/src/residencyCache.ts new file mode 100644 index 00000000000..4792e3f6940 --- /dev/null +++ b/internal-packages/run-store/src/residencyCache.ts @@ -0,0 +1,53 @@ +import { LRUCache } from "lru-cache"; + +export type Residency = "resident" | "non-resident"; + +/** Entries, not bytes. At the default this is roughly 25 MB of run ids. */ +const DEFAULT_MAX = 250_000; + +/** + * Per-process memory of which runs the mirror owns, so the hot path stops asking Redis. + * + * The keyspace is a run's residency record and it lives in Redis, so learning that a run is NOT + * resident used to cost a round trip, once per transition, forever. Under a brownout that is the + * full retry budget for a question whose answer never changes. + * + * Soundness rests on residency being monotonic. Only a birth creates a keyspace: the append script + * refuses `kind: "transition"` into a dead one, and the repair appends as a transition, so nothing + * else can mint one. Therefore: + * + * - `non-resident` is PERMANENT, and may be trusted to skip the network entirely. + * - `resident` is a hint. A keyspace can still go away under a completion expiry, a sweep, or an + * eviction, so a stale positive costs one round trip that returns `skippedNoKeyspace`. That is + * the safe direction, and it is why the cache never needs a TTL. + * + * There is no negative-cache invalidation and there must not be one. A run that was told it has no + * keyspace has to keep that answer for life, or it would change stores half way through. + */ +export class ResidencyCache { + readonly #entries: LRUCache; + + constructor(options: { max?: number } = {}) { + this.#entries = new LRUCache({ max: options.max ?? DEFAULT_MAX }); + } + + get(runId: string): Residency | undefined { + return this.#entries.get(runId); + } + + /** Hint only. Refused once the run is known non-resident, because that answer is final. */ + setResident(runId: string): void { + if (this.#entries.get(runId) === "non-resident") { + return; + } + this.#entries.set(runId, "resident"); + } + + setNonResident(runId: string): void { + this.#entries.set(runId, "non-resident"); + } + + get size(): number { + return this.#entries.size; + } +} diff --git a/internal-packages/run-store/src/snapshotEntry.ts b/internal-packages/run-store/src/snapshotEntry.ts index 748a95f6d1a..738685a7e2f 100644 --- a/internal-packages/run-store/src/snapshotEntry.ts +++ b/internal-packages/run-store/src/snapshotEntry.ts @@ -166,3 +166,54 @@ export function entryFromCreateExecutionSnapshot( export function isTerminalEntry(entry: SnapshotEntryInput): boolean { return entry.executionStatus === "FINISHED"; } + +/** + * Builds an entry from a snapshot ROW rather than a write site's input, which only the repair path + * needs: it re-appends a snapshot Postgres already holds, so every derived value is read back rather + * than reproduced. A null column is omitted, not carried as null, so the document matches what the + * lost append would have written. + */ +export function entryFromSnapshotRow(row: SnapshotRowForEntry): SnapshotEntryInput { + return { + id: row.id, + runId: row.runId, + createdAt: row.createdAt.toISOString(), + engine: "V2", + executionStatus: row.executionStatus, + description: row.description, + runStatus: snapshotRunStatus(row.runStatus), + ...(row.attemptNumber !== null && { attemptNumber: row.attemptNumber }), + ...(row.previousSnapshotId !== null && { previousSnapshotId: row.previousSnapshotId }), + ...(row.batchId !== null && { batchId: row.batchId }), + environmentId: row.environmentId, + environmentType: row.environmentType, + projectId: row.projectId, + organizationId: row.organizationId, + ...(row.checkpointId !== null && { checkpointId: row.checkpointId }), + ...(row.workerId !== null && { workerId: row.workerId }), + ...(row.runnerId !== null && { runnerId: row.runnerId }), + ...(row.metadata !== null && { metadata: row.metadata }), + ...(row.error !== null && { error: row.error }), + }; +} + +export type SnapshotRowForEntry = { + id: string; + runId: string; + createdAt: Date; + executionStatus: string; + description: string; + runStatus: TaskRunStatus; + attemptNumber: number | null; + previousSnapshotId: string | null; + batchId: string | null; + environmentId: string; + environmentType: string; + projectId: string; + organizationId: string; + checkpointId: string | null; + workerId: string | null; + runnerId: string | null; + metadata: unknown; + error: string | null; +}; diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts index 7d7eb2aca0d..6a1e10c82d9 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.test.ts @@ -466,4 +466,55 @@ describe("SnapshotOrphanSweeper", () => { await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); } }); + containerTest( + "one wrong-typed keyspace costs that run its pass, not every run's", + async ({ prisma, redisOptions }) => { + // A single malformed keyspace used to abort the WHOLE pass: the Redis commands in the two + // rule bodies were uncaught, so a WRONGTYPE propagated out of sweep() and every later pass + // hit the same key. Collection stopped for every run until a human deleted it. + const store = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const runStore = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const sweeper = new SnapshotOrphanSweeper({ + redisOptions, + runStore: runStore as unknown as RunStore, + completedTtlMs: COMPLETED_TTL_MS, + orphanAgeMs: ORPHAN_AGE_MS, + confirmOrphanAfterMs: 0, + }); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + + // A healthy terminal run that rule 1 must still expire. + const healthyId = generateInternalId(); + await store.append({ + entry: birthEntry(healthyId, env, new Date()), + kind: "birth", + isTerminal: false, + }); + await prisma.taskRun.create({ + data: { ...buildCreateRunData(healthyId, env), status: "COMPLETED_SUCCESSFULLY" }, + }); + + // A keyspace whose `seq` is a string where the code expects a hash. Discovered by the scan + // (it matches on `:e`), then every command against `seq` raises WRONGTYPE. + const brokenId = generateInternalId(); + const brokenKeys = snapshotKeys(brokenId); + await probe.hset(brokenKeys.e, "s_x", JSON.stringify({ id: "s_x", runId: brokenId })); + await probe.set(brokenKeys.seq, "not-a-hash"); + + const result = await sweeper.sweep(); + + // The pass completes and reports the failure rather than throwing it. + expect(result.failed).toBe(1); + expect(result.partial).toBe(false); + + // And the healthy run was still collected, which is the whole point. + expect(result.expired).toBe(1); + expect(await probe.pttl(snapshotKeys(healthyId).e)).toBeGreaterThan(0); + } finally { + await Promise.all([store.quit(), sweeper.quit(), probe.quit().catch(() => {})]); + } + } + ); }); diff --git a/internal-packages/run-store/src/snapshotOrphanSweeper.ts b/internal-packages/run-store/src/snapshotOrphanSweeper.ts index 86bb084d5c9..0e3cb25f248 100644 --- a/internal-packages/run-store/src/snapshotOrphanSweeper.ts +++ b/internal-packages/run-store/src/snapshotOrphanSweeper.ts @@ -80,6 +80,13 @@ export type SweepResult = { * on an observed sweep pass, so the observation has to carry its own coverage. */ nodes: number; + /** + * Keyspaces whose own Redis commands raised, so the pass acted on neither rule for them. Contained + * per run on purpose: a single malformed keyspace used to abort the whole pass, and because the + * scan restarts from the same place every time, collection then stopped for EVERY run until a + * human removed the key. One bad keyspace now costs itself one pass. + */ + failed: number; /** True when the pass stopped early on its deadline or abort signal, so coverage is incomplete. */ partial: boolean; }; @@ -200,6 +207,7 @@ export class SnapshotOrphanSweeper { deleted: 0, skipped: 0, pendingDeletion: 0, + failed: 0, nodes: 0, partial: false, }; @@ -295,18 +303,31 @@ export class SnapshotOrphanSweeper { for (const runId of runIds) { const run = rows.get(runId); - if (!run) { - await this.#applyRuleTwo(runId, dryRun, result); - continue; - } + // Per run, not per batch. The rule bodies below issue their own Redis commands, and an error + // from any of them used to leave the whole pass. The run-row lookup above has had its own + // guard from the start; this gives the Redis half the same treatment, for the same reason. + try { + if (!run) { + await this.#applyRuleTwo(runId, dryRun, result); + continue; + } - if (!FINAL.has(run.status)) { - // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. - result.skipped += 1; - continue; - } + if (!FINAL.has(run.status)) { + // A live run. A SUSPENDED run can legitimately wait for weeks, so this is never touched. + result.skipped += 1; + continue; + } - await this.#applyRuleOne(runId, dryRun, result); + await this.#applyRuleOne(runId, dryRun, result); + } catch (error) { + // Never rethrown. A keyspace this pass cannot read is a keyspace the NEXT pass can try, and + // the alternative is losing collection for every other run as well. + this.#logger.error("SnapshotOrphanSweeper skipped a keyspace after a failed command", { + runId, + error, + }); + result.failed += 1; + } } } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.birthFatality.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.birthFatality.test.ts new file mode 100644 index 00000000000..fa34c908e8f --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.birthFatality.test.ts @@ -0,0 +1,89 @@ +// A failed birth append is fatal on the ORGANISATION dial, not the global position. Before +// redis-only Postgres is authoritative, so a lost birth is survivable and run creation proceeds; +// at redis-only Postgres writes no snapshot, so a run born without its Redis snapshot would have +// none anywhere and the append must throw before the run row exists so the caller retries clean. +// +// The load-bearing case is the second test: the global dial is redis-only while THIS org is still +// dual-write. A regression to the global position (`this.mode`) would throw here and wrongly fail +// run creation for an org whose own position still has Postgres authoritative. +import { describe, expect, it } from "vitest"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, + type SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +const ORG = "org_a"; + +function harness(opts: { global: SnapshotStoreMode; forOrg: SnapshotStoreMode }) { + const delegateCalls: string[] = []; + + // Every append rejects with a NON-injected error, so the retry loop exhausts and hits the + // terminal branch. An injected fault would mean "the process died", which is a different path. + const redis = new Proxy({} as RedisSnapshotStore, { + get: (_t, prop) => { + if (prop === "append") { + return () => Promise.reject(new Error("redis append boom")); + } + return () => Promise.resolve({ outcome: "written", seq: 1 }); + }, + }); + + const delegate = new Proxy({} as Record, { + get: + (_t, prop) => + (...__: unknown[]) => { + delegateCalls.push(String(prop)); + return Promise.resolve({}); + }, + }) as unknown as RunStore; + + const modeResolver: SnapshotStoreModeResolver = { + resolve: (organizationId?: string) => + organizationId === undefined ? opts.global : opts.forOrg, + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.global, + modeResolver, + }); + return { decorated, delegateCalls }; +} + +function createRunParams() { + return { + data: { id: "run_1" }, + snapshot: { + id: "snap_1", + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: "env_1", + environmentType: "PRODUCTION" as const, + projectId: "proj_1", + organizationId: ORG, + }, + } as never; +} + +describe("birth append fatality is decided by the organisation dial", () => { + it("throws when the org resolves to redis-only, so run creation retries clean", async () => { + // Global is dual-write; only the org is redis-only. The throw must follow the org, not global. + const { decorated, delegateCalls } = harness({ global: "dual-write", forOrg: "redis-only" }); + + await expect(decorated.createRun(createRunParams())).rejects.toThrow(); + // No run row when the birth is fatal. + expect(delegateCalls).toEqual([]); + }); + + it("does NOT throw when the org is dual-write even though the global dial is redis-only", async () => { + // The org-scoping guard: global redis-only would throw if the terminal branch read `this.mode`. + const { decorated, delegateCalls } = harness({ global: "redis-only", forOrg: "dual-write" }); + + await expect(decorated.createRun(createRunParams())).resolves.toBeDefined(); + expect(delegateCalls).toContain("createRun"); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.birthWarm.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.birthWarm.test.ts new file mode 100644 index 00000000000..f80d019aa55 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.birthWarm.test.ts @@ -0,0 +1,138 @@ +// The per-organisation dial is served from a short-lived cache, and on a miss the resolver answers +// with the DEPLOYMENT-WIDE position and warms the cache off the request path. For reads that is the +// right trade. For a BIRTH it is not: residency is fixed at birth and permanent, so a run born +// during a cache miss is excluded from the mirror for its whole life. +// +// Observed live: three runs born back to back were all resident, then after a 14 minute idle gap the +// next run was not, because the cache entry had expired. A cache miss is not a rare event, it is any +// gap longer than the cache lifetime, so on bursty traffic the first run of every burst was lost. +// +// So a birth waits for the organisation's real answer. Transitions do NOT: they stay dial-blind and +// synchronous, which is what stops a run changing stores half way through its life. +import { describe, expect, it } from "vitest"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { + SnapshotStoreMode, + SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +const SCOPE = { + environmentId: "env_1", + environmentType: "PRODUCTION", + projectId: "proj_1", + organizationId: "org_1", +} as const; + +function harness(opts: { + /** What resolve() answers BEFORE warm() has run. Models the cache-miss fallback. */ + cold: SnapshotStoreMode; + /** What resolve() answers AFTER warm() has run. Models the organisation's real value. */ + warmed: SnapshotStoreMode; + /** Omit to model a resolver that offers no warm at all. */ + offerWarm?: boolean; + /** Make warm() reject, to prove a birth still proceeds. */ + warmThrows?: boolean; +}) { + const appends: { kind: string }[] = []; + let warmCalls = 0; + let isWarm = false; + + const redis = { + append: async (args: { kind: string }) => { + appends.push({ kind: args.kind }); + return { outcome: "written" as const, seq: appends.length }; + }, + } as unknown as RedisSnapshotStore; + + const delegate = new Proxy({} as Record, { + get: () => () => Promise.resolve({}), + }) as unknown as RunStore; + + const modeResolver: SnapshotStoreModeResolver = { + resolve: () => (isWarm ? opts.warmed : opts.cold), + ...(opts.offerWarm !== false && { + warm: async () => { + warmCalls += 1; + if (opts.warmThrows) throw new Error("flag read failed"); + isWarm = true; + }, + }), + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.cold, + modeResolver, + }); + + return { decorated, appends, warmCalls: () => warmCalls }; +} + +function birthParams() { + return { + data: { id: "run_1" } as never, + snapshot: { + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run created", + runStatus: "PENDING" as const, + ...SCOPE, + }, + }; +} + +describe("a birth waits for the organisation's real dial", () => { + it("mirrors when the cache was cold but the organisation is opted in", async () => { + // The case that was losing runs: cold cache answers with the global `off`, the organisation is + // actually at dual-write. Without warming, this birth is non-resident forever. + const h = harness({ cold: "off", warmed: "dual-write" }); + + await h.decorated.createRun(birthParams()); + + expect(h.warmCalls()).toBe(1); + expect(h.appends).toEqual([{ kind: "birth" }]); + }); + + it("does not mirror when the organisation really is off", async () => { + // Warming must fetch the truth, not force a mirror. + const h = harness({ cold: "dual-write", warmed: "off" }); + + await h.decorated.createRun(birthParams()); + + expect(h.warmCalls()).toBe(1); + expect(h.appends).toEqual([]); + }); + + it("warms the cancelled-run birth path too", async () => { + const h = harness({ cold: "off", warmed: "dual-write" }); + + await h.decorated.createCancelledRun(birthParams()); + + expect(h.warmCalls()).toBe(1); + expect(h.appends).toEqual([{ kind: "birth" }]); + }); + + it("still births when the warm read fails, falling back to the position it already had", async () => { + // A flag read that fails or times out must never fail a trigger. The old behaviour is the + // fallback, not an error. + const h = harness({ cold: "dual-write", warmed: "off", warmThrows: true }); + + await expect(h.decorated.createRun(birthParams())).resolves.toBeDefined(); + + expect(h.warmCalls()).toBe(1); + // Cold answer stood, because the warm never landed. + expect(h.appends).toEqual([{ kind: "birth" }]); + }); + + it("works against a resolver that offers no warm at all", async () => { + // The seam is optional, so every existing resolver and test double keeps working. + const h = harness({ cold: "dual-write", warmed: "off", offerWarm: false }); + + await h.decorated.createRun(birthParams()); + + expect(h.warmCalls()).toBe(0); + expect(h.appends).toEqual([{ kind: "birth" }]); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.checkpoint.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.checkpoint.test.ts new file mode 100644 index 00000000000..b6c6295a736 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.checkpoint.test.ts @@ -0,0 +1,152 @@ +// Gate 14. A suspended run's snapshot points at a checkpoint, and that is how the run is resumed. +// The entry in Redis carries only `checkpointId`: the checkpoint ROW stays in Postgres and is read +// back through the delegate, and only when the entry says one exists, so the common read of a +// running run with no checkpoint costs no Postgres call at all. +// +// That split is the thing worth testing. A Redis-served snapshot that dropped its checkpoint, or +// returned it in a different shape than the Postgres read, would resume a run with nowhere to +// restore from, and the mirror would look healthy while doing it. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function birthSnapshot(env: SnapshotFixtureEnv) { + return { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +describe("a checkpoint on a snapshot served from Redis", () => { + containerTest( + "comes back, and matches what Postgres would have returned", + async ({ prisma, redisOptions }) => { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const plain = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const decorated = new TaskRunExecutionSnapshotStore(plain as unknown as RunStore, { + store: redis, + mode: "redis-read", + readPercent: 100, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + const checkpoint = await prisma.taskRunCheckpoint.create({ + data: { + friendlyId: `checkpoint_${generateInternalId()}`, + type: "DOCKER", + location: "s3://bucket/checkpoint.tar", + imageRef: "registry/image@sha256:abc", + reason: "wait for duration", + projectId: env.projectId, + runtimeEnvironmentId: env.id, + }, + }); + + // A suspend transition: the snapshot names the checkpoint the run must restore from. + await decorated.createExecutionSnapshot({ + run: { id: runId, status: "WAITING_TO_RESUME" }, + snapshot: { + executionStatus: "SUSPENDED", + description: "Run was suspended", + }, + // Top level, not inside `snapshot`. + checkpointId: checkpoint.id, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + } as never); + + // Served from Redis, because the dial is at redis-read and the run is resident. + const fromRedis = await decorated.findLatestExecutionSnapshot(runId); + // The same question asked of Postgres alone, as the oracle. + const fromPostgres = await plain.findLatestExecutionSnapshot(runId); + + expect(fromRedis).not.toBeNull(); + expect(fromRedis!.executionStatus).toBe("SUSPENDED"); + + // The identifier survived the trip through Redis. + expect(fromRedis!.checkpointId).toBe(checkpoint.id); + + // And the ROW was re-attached, not just the id. A resume needs the location and the image. + expect(fromRedis!.checkpoint).not.toBeNull(); + expect(fromRedis!.checkpoint!.id).toBe(checkpoint.id); + expect(fromRedis!.checkpoint!.location).toBe("s3://bucket/checkpoint.tar"); + expect(fromRedis!.checkpoint!.imageRef).toBe("registry/image@sha256:abc"); + + // The claim that matters: the two stores answer identically. + expect(fromRedis!.checkpoint).toEqual(fromPostgres!.checkpoint); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "costs no Postgres read when the snapshot has no checkpoint", + async ({ prisma, redisOptions }) => { + // The other half of the split. Hydrating unconditionally would put a Postgres read back on + // the hot path of every running run, which is what the entry's own checkpointId avoids. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const plain = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + // Observed through the store's own metrics seam rather than by replacing a method on the + // production store. `recordRead` reports which store served each read, which is exactly the + // question, and it is an injection point the class already offers. + const reads: { method: string; servedBy: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore(plain as unknown as RunStore, { + store: redis, + mode: "redis-read", + readPercent: 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, servedBy) => reads.push({ method, servedBy }), + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + reads.length = 0; + const read = await decorated.findLatestExecutionSnapshot(runId); + + expect(read).not.toBeNull(); + expect(read!.checkpoint).toBeNull(); + // Served entirely by Redis. Hydrating unconditionally would put a Postgres read back on the + // hot path of every running run, which is the cost the entry's own checkpointId avoids. + expect(reads).toEqual([{ method: "findLatestExecutionSnapshot", servedBy: "redis" }]); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.hardStop.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.hardStop.test.ts new file mode 100644 index 00000000000..777e25515c7 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.hardStop.test.ts @@ -0,0 +1,68 @@ +// `off` is a rollout position, not a kill switch: refusing a resident run's transitions freezes its +// Redis head while Postgres advances, which is the mid-life store switch residency forbids. So +// `off` stops new residency only, and the halt switch is the hard stop. Pure predicates, no +// containers. +import { describe, expect, it } from "vitest"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { + SnapshotStoreMode, + SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string): boolean }; + +function storeWith(options: { + mode: SnapshotStoreMode; + resolver?: SnapshotStoreModeResolver; + halted?: boolean; +}) { + return new TaskRunExecutionSnapshotStore({} as unknown as RunStore, { + store: {} as never, + mode: options.mode, + ...(options.resolver && { modeResolver: options.resolver }), + ...(options.halted !== undefined && { halted: () => options.halted === true }), + readPercent: 100, + }); +} + +describe("a global dial-down to off", () => { + it("keeps a resident run's transitions mirroring", () => { + const s = storeWith({ mode: "off" }); + expect(s.writesRedisForTransitionTest()).toBe(true); + }); + + it("stops new births", () => { + const s = storeWith({ mode: "off" }); + expect(s.writesRedisForBirthTest("org_1")).toBe(false); + expect(s.writesRedisForBirthTest(undefined)).toBe(false); + }); +}); + +describe("the halt switch", () => { + it("stops births and transitions at every dial position", () => { + for (const mode of ["off", "dual-write", "redis-read", "redis-only"] as const) { + const s = storeWith({ mode, halted: true }); + expect(s.writesRedisForBirthTest("org_1")).toBe(false); + expect(s.writesRedisForTransitionTest()).toBe(false); + } + }); + + it("leaves births and transitions alone when it is not thrown", () => { + const s = storeWith({ mode: "dual-write", halted: false }); + expect(s.writesRedisForBirthTest("org_1")).toBe(true); + expect(s.writesRedisForTransitionTest()).toBe(true); + }); + + it("sends redis-read reads back to Postgres, which is still authoritative there", () => { + const s = storeWith({ mode: "redis-read", halted: true }) as unknown as CohortProbe; + expect(s.readsFromRedis("run_halted")).toBe(false); + }); + + it("leaves redis-only reads on Redis, which holds the only copy", () => { + // Routing these to Postgres reads nothing at all: at this position Postgres holds no snapshot + // rows. A halt here is a resync, not a fallback. + const s = storeWith({ mode: "redis-only", halted: true }) as unknown as CohortProbe; + expect(s.readsFromRedis("run_halted")).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.hotPath.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.hotPath.test.ts new file mode 100644 index 00000000000..040c2d73802 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.hotPath.test.ts @@ -0,0 +1,429 @@ +// Problem: at `off` the store still put Redis on the run path for EVERY run. Births were dial-gated, +// but transitions were not, and the residency test lives inside the Lua script, so the store had to +// complete a round trip just to learn a run was not its own. Measured live: 2 percent with a healthy +// Redis, 4x (34187ms against 8258ms) under a brownout, for every run, with no decay as resident runs +// drained. +// +// These tests count actual Redis commands, because that is the claim. A latency assertion would pass +// on a fast local Redis while the round trip was still there. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { createRedisClient } from "@internal/redis"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +/** + * Script calls the server has served. Counts EVAL as well as EVALSHA: ioredis sends EVALSHA and + * falls back to EVAL when the server has not cached the script yet, so counting one of them alone + * reads as zero on a cold server and the assertion passes for the wrong reason. + */ +async function scriptCalls(probe: { info: (section: string) => Promise }): Promise { + const stats = await probe.info("commandstats"); + const of = (cmd: string) => + Number(new RegExp(`cmdstat_${cmd}:calls=(\\d+)`).exec(stats)?.[1] ?? 0); + return of("eval") + of("evalsha"); +} + +function birthSnapshot(env: SnapshotFixtureEnv) { + return { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; +} + +function completionInput(env: SnapshotFixtureEnv) { + return { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }; +} + +/** + * A real store whose waitpoint-id lookup fails. Used to make the SECOND Redis call of a read fail + * after the first has succeeded, which is the only way to reach the hydration fallback. + */ +class HydrationFailingStore extends RedisSnapshotStore { + hydrationCalls = 0; + + override async getSnapshotWaitpointIds( + ...args: Parameters + ): ReturnType { + this.hydrationCalls += 1; + void args; + throw new Error("Command timed out"); + } +} + +describe("Redis stays off the hot path for a non-resident run", () => { + containerTest( + "a run born at off probes once, then never again", + async ({ prisma, redisOptions }) => { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode: "off" } + ); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + // Nothing is assumed from the birth decision. A birth path can be re-entered, so a local + // "did not mirror" is not proof of anything; only the script's reply is. + expect(redis.residencyFor(runId)).toBeUndefined(); + + const before = await scriptCalls(probe); + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + const mid = await scriptCalls(probe); + + // One probe, which is what makes the answer authoritative. + expect(mid - before).toBe(1); + expect(redis.residencyFor(runId)).toBe("non-resident"); + + // The assertion that matters. Before the cache this was one script call per transition, for + // the life of every run, and under a brownout each one cost the full retry budget. + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + expect((await scriptCalls(probe)) - mid).toBe(0); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); + + containerTest( + "a resident run still mirrors, so the cache cannot be a blanket off switch", + async ({ prisma, redisOptions }) => { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode: "dual-write" } + ); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + expect(redis.residencyFor(runId)).toBe("resident"); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + + const head = await redis.getLatest(runId); + expect(head?.entry.executionStatus).toBe("FINISHED"); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "a probe that finds no keyspace is remembered, so it happens once and not once per transition", + async ({ prisma, redisOptions }) => { + // The cold-cache case: a process that did not see the birth. It probes once, learns, and stops. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode: "dual-write" } + ); + const probe = createRedisClient(redisOptions, { onError: () => {} }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + // A run row with no keyspace, exactly as a run born elsewhere at off would look. + await prisma.taskRun.create({ data: buildCreateRunData(runId, env) }); + expect(redis.residencyFor(runId)).toBeUndefined(); + + const first = await scriptCalls(probe); + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + const second = await scriptCalls(probe); + + // One probe, and it taught the cache. + expect(second - first).toBe(1); + expect(redis.residencyFor(runId)).toBe("non-resident"); + + await decorated.completeAttemptSuccess(runId, completionInput(env), { + select: { id: true }, + }); + const third = await scriptCalls(probe); + expect(third - second).toBe(0); + } finally { + await Promise.all([redis.quit(), probe.quit().catch(() => {})]); + } + } + ); +}); + +describe("a Redis that stops answering", () => { + containerTest( + "serves reads from Postgres instead of throwing into the engine", + async ({ prisma, redisOptions }) => { + // The read paths fell back on a miss and on a dangling cycle, but not on an ERROR, so a + // brownout at redis-read turned an engine read into a throw once the command timed out. + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode: "redis-read", readPercent: 100 } + ); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + // Reads work while Redis answers. + expect(await decorated.findLatestExecutionSnapshot(runId)).not.toBeNull(); + + // Now take Redis away underneath it, the way a brownout does. + await redis.quit(); + + const served = await decorated.findLatestExecutionSnapshot(runId); + expect(served).not.toBeNull(); + expect(served!.runId).toBe(runId); + } finally { + await redis.quit(); + } + } + ); + + containerTest("stops calling out once the circuit opens", async ({ prisma, redisOptions }) => { + const redis = new RedisSnapshotStore({ + redisOptions, + completedTtlMs: COMPLETED_TTL_MS, + breaker: { failureThreshold: 2, openDurationMs: 60_000 }, + }); + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { store: redis, mode: "redis-read", readPercent: 100 } + ); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + await redis.quit(); + expect(redis.breakerState).toBe("closed"); + + for (let i = 0; i < 4; i++) { + expect(await decorated.findLatestExecutionSnapshot(runId)).not.toBeNull(); + } + + // Open, so every later call is refused locally rather than waiting out another timeout. This + // is what bounds the cold-cache probe under a brownout. + expect(redis.breakerState).toBe("open"); + } finally { + await redis.quit(); + } + }); + containerTest( + "falls back even when the SECOND Redis call is the one that fails", + async ({ prisma, redisOptions }) => { + // The first fix caught the read itself but stopped before hydration. A snapshot with a wait + // cycle whose ids were not carried by the read makes a follow-up Redis call inside #hydrate, + // and a failure there threw straight into the engine at redis-read, which is the case the + // fallback exists for. + // + // A subclass rather than a replaced property: this is a real store on a real connection, so + // every other command still goes through the production path and the real cluster. Only the + // one command under test is specialised, because the scenario needs a SECOND Redis call to + // fail after a first has already succeeded, and no seam exposes that. The write-path fault + // injector models process crashes, not command failures, so it is the wrong tool here. + const redis = new HydrationFailingStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const plain = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const decorated = new TaskRunExecutionSnapshotStore(plain as unknown as RunStore, { + store: redis, + mode: "redis-read", + readPercent: 100, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + // A transition carrying a wait cycle, so the head has waitpoints and hydration has a reason + // to ask Redis for them. + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${generateInternalId()}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + environmentId: env.id, + projectId: env.projectId, + }, + }); + await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING" }, + snapshot: { executionStatus: "EXECUTING", description: "Resumed" }, + completedWaitpoints: [{ id: waitpoint.id, index: 0 }], + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + } as never); + + // One more transition, so the waitpoint-bearing entry is no longer the head. Only the HEAD + // row of a window is decoded with its waitpoint ids; every other row has to ask, and that + // ask is the second Redis call this test is about. + await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING" }, + snapshot: { executionStatus: "EXECUTING", description: "Still executing" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + } as never); + + // The WINDOW read, not the head read. The head read carries the waitpoint ids already, so + // its hydration never asks Redis; the window entries do not, which is where the second call + // lives and where the gap was. + const served = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 3_600_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 10, + } as never); + + expect(Array.isArray(served)).toBe(true); + expect(served.length).toBeGreaterThan(0); + + // Asserted, not hoped for: without this the test passes on a run whose hydration never + // reaches Redis, and proves nothing at all. + expect(redis.hydrationCalls).toBeGreaterThanOrEqual(1); + } finally { + await redis.quit(); + } + } + ); + containerTest( + "records the read source ONCE, not once per attempt", + async ({ prisma, redisOptions }) => { + // Adding the hydration fallback after the recordRead call meant one logical read incremented + // BOTH series: `redis` on the way in, then `postgres` when hydration fell back. Any dashboard + // built on read_source then over-counts, and the redis/postgres split stops summing to the + // number of reads. + const redis = new HydrationFailingStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const plain = new PostgresRunStore({ prisma, readOnlyPrisma: prisma }); + const reads: { method: string; servedBy: string }[] = []; + const decorated = new TaskRunExecutionSnapshotStore(plain as unknown as RunStore, { + store: redis, + mode: "redis-read", + readPercent: 100, + metrics: { + recordWrite: () => {}, + recordAppendFailed: () => {}, + recordRead: (method, servedBy) => reads.push({ method, servedBy }), + }, + }); + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await decorated.createRun({ + data: buildCreateRunData(runId, env), + snapshot: birthSnapshot(env), + }); + + const waitpoint = await prisma.waitpoint.create({ + data: { + friendlyId: `waitpoint_${generateInternalId()}`, + type: "MANUAL", + status: "COMPLETED", + completedAt: new Date(), + idempotencyKey: generateInternalId(), + userProvidedIdempotencyKey: false, + environmentId: env.id, + projectId: env.projectId, + }, + }); + await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING" }, + snapshot: { executionStatus: "EXECUTING", description: "Resumed" }, + completedWaitpoints: [{ id: waitpoint.id, index: 0 }], + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + } as never); + await decorated.createExecutionSnapshot({ + run: { id: runId, status: "EXECUTING" }, + snapshot: { executionStatus: "EXECUTING", description: "Still executing" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + } as never); + + reads.length = 0; + const served = await decorated.findManyExecutionSnapshots({ + where: { runId, isValid: true, createdAt: { gt: new Date(Date.now() - 3_600_000) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 10, + } as never); + + expect(served.length).toBeGreaterThan(0); + // Hydration failed, so Postgres served it, and that is the ONLY thing recorded. + expect(reads).toEqual([{ method: "findManyExecutionSnapshots", servedBy: "postgres" }]); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.latch.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.latch.test.ts new file mode 100644 index 00000000000..8bd11eb5bfa --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.latch.test.ts @@ -0,0 +1,163 @@ +// `off` was not inert. A transition must ask whether its run is resident, and the keyspace is the +// only record of that, so at `off` AFTER a ramp every transition still has to ask or a resident +// run's head freezes while Postgres moves on. +// +// A transition may therefore be skipped only when nothing of this org's could be resident. A run is +// resident only if its org's dial was non-off at its birth, i.e. the global dial had ever gone +// non-off OR the org itself was ever enabled. So the skip is sound only when the global dial has +// NEVER been non-off (globalModeEverEnabled() === false) AND this org is DEFINITELY never-enabled +// (orgDefinitelyNeverEnabled(org) === true). Any uncertainty falls to "probe". +import { describe, expect, it } from "vitest"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { SnapshotStoreModeResolver } from "./taskRunExecutionSnapshotStore.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +const SCOPE = { + environmentId: "env_1", + environmentType: "PRODUCTION", + projectId: "proj_1", + organizationId: "org_1", +} as const; + +function harness(opts: { + mode: "off" | "dual-write"; + globalModeEverEnabled?: boolean; + orgDefinitelyNeverEnabled?: (organizationId: string) => boolean; +}) { + const touched: string[] = []; + const redis = new Proxy({} as RedisSnapshotStore, { + get: (_t, prop) => { + return (...__: unknown[]) => { + touched.push(String(prop)); + return Promise.resolve({ outcome: "written", seq: 1 }); + }; + }, + }); + const delegate = new Proxy({} as Record, { + get: () => () => Promise.resolve({}), + }) as unknown as RunStore; + + const modeResolver: SnapshotStoreModeResolver = { + resolve: () => opts.mode, + ...(opts.globalModeEverEnabled !== undefined && { + globalModeEverEnabled: () => opts.globalModeEverEnabled!, + }), + ...(opts.orgDefinitelyNeverEnabled && { + orgDefinitelyNeverEnabled: opts.orgDefinitelyNeverEnabled, + }), + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.mode, + modeResolver, + }); + return { decorated, touched }; +} + +/** A completion whose snapshot carries a chosen organisation id. */ +function completionForOrg(organizationId: string) { + return { ...completion, snapshot: { ...completion.snapshot, organizationId } }; +} + +const completion = { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED" as const, + description: "done", + runStatus: "COMPLETED_SUCCESSFULLY" as const, + attemptNumber: 1, + ...SCOPE, + }, +}; + +describe("the sound transition skip", () => { + it("touches the store NOT AT ALL for a definitely-never org while the global dial has never moved", async () => { + // The assertion the whole change exists for: no keyspace can exist for this org, so no probe. + const h = harness({ + mode: "dual-write", + globalModeEverEnabled: false, + orgDefinitelyNeverEnabled: () => true, + }); + + await h.decorated.completeAttemptSuccess("run_1", completionForOrg("org_a"), { + select: { id: true }, + }); + + expect(h.touched).toEqual([]); + }); + + it("still probes an org that has ever been enabled", async () => { + // orgDefinitelyNeverEnabled is false for an org that may hold resident runs, so it keeps asking. + const h = harness({ + mode: "dual-write", + globalModeEverEnabled: false, + orgDefinitelyNeverEnabled: (org) => org !== "org_a", + }); + + await h.decorated.completeAttemptSuccess("run_1", completionForOrg("org_a"), { + select: { id: true }, + }); + + expect(h.touched).toContain("append"); + }); + + it("never skips once the global dial has ever been non-off, even for a definitely-never org", async () => { + // Non-negotiable. Once the global dial has moved, resident runs exist, and suppressing their + // transitions freezes a head while Postgres advances. The global latch beats the per-org one. + const h = harness({ + mode: "off", + globalModeEverEnabled: true, + orgDefinitelyNeverEnabled: () => true, + }); + + await h.decorated.completeAttemptSuccess("run_1", completionForOrg("org_a"), { + select: { id: true }, + }); + + expect(h.touched).toContain("append"); + }); + + it("never skips when the organisation is undefined", () => { + // No org id means orgDefinitelyNeverEnabled cannot be consulted, so an unknown org errs toward + // asking. The per-org term is only reached once an org id is supplied. + const h = harness({ + mode: "dual-write", + globalModeEverEnabled: false, + orgDefinitelyNeverEnabled: () => true, + }); + + expect(h.decorated.writesRedisForTransitionTest()).toBe(true); + expect(h.decorated.writesRedisForTransitionTest("org_a")).toBe(false); + }); + + it("asks when the resolver offers no latch signals, so an unwired deployment is unchanged", async () => { + const h = harness({ mode: "off" }); + + await h.decorated.completeAttemptSuccess("run_1", completion, { select: { id: true } }); + + expect(h.touched).toContain("append"); + }); + + it("asks when only one of the two signals permits a skip", () => { + // Both halves are required. A cold census (orgDefinitelyNeverEnabled false) with an unmoved + // global dial must still probe, and vice versa. + const globalOnly = harness({ + mode: "off", + globalModeEverEnabled: false, + orgDefinitelyNeverEnabled: () => false, + }); + expect(globalOnly.decorated.writesRedisForTransitionTest("org_a")).toBe(true); + + const orgOnly = harness({ + mode: "off", + globalModeEverEnabled: true, + orgDefinitelyNeverEnabled: () => true, + }); + expect(orgOnly.decorated.writesRedisForTransitionTest("org_a")).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts new file mode 100644 index 00000000000..47bf16d5676 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.modeResolver.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, + type SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; + +function storeWith(options: { + mode?: SnapshotStoreMode; + modeResolver?: SnapshotStoreModeResolver; +}) { + return new TaskRunExecutionSnapshotStore({} as never, { + store: {} as never, + ...options, + }); +} + +function resolverOf(perOrg: Record, global: SnapshotStoreMode) { + return { resolve: (orgId?: string) => (orgId ? (perOrg[orgId] ?? global) : global) }; +} + +// These assert the per-organisation contract, which now governs BIRTHS only: a birth fixes the +// run's store for life. Transitions deliberately ignore the organisation, and that is asserted in +// taskRunExecutionSnapshotStore.residency.test.ts. +describe("TaskRunExecutionSnapshotStore mode resolution", () => { + it("prefers the resolver over the static mode", () => { + const store = storeWith({ mode: "off", modeResolver: resolverOf({}, "dual-write") }); + expect(store.mode).toBe("dual-write"); + }); + + it("falls back to the static mode when no resolver is supplied", () => { + expect(storeWith({ mode: "redis-read" }).mode).toBe("redis-read"); + }); + + it("defaults to off with neither", () => { + expect(storeWith({}).mode).toBe("off"); + }); + + it("resolves per organisation", () => { + const store = storeWith({ modeResolver: resolverOf({ org_a: "dual-write" }, "off") }); + expect(store.writesRedisForBirthTest("org_a")).toBe(true); + expect(store.writesRedisForBirthTest("org_b")).toBe(false); + }); + + it("lets an organisation be off while the global answer is on", () => { + const store = storeWith({ modeResolver: resolverOf({ org_a: "off" }, "dual-write") }); + expect(store.writesRedisForBirthTest("org_a")).toBe(false); + expect(store.writesRedisForBirthTest("org_b")).toBe(true); + }); + + it("sees a resolver answer that changes after construction", () => { + let current: SnapshotStoreMode = "off"; + const store = storeWith({ modeResolver: { resolve: () => current } }); + expect(store.mode).toBe("off"); + current = "dual-write"; + expect(store.mode).toBe("dual-write"); + }); + + it("resolves the global answer when no organisation is supplied", () => { + const store = storeWith({ modeResolver: resolverOf({ org_a: "off" }, "redis-read") }); + expect(store.writesRedisForBirthTest()).toBe(true); + }); + + it("resolves the fatal-birth decision per organisation, not globally", () => { + // A lost birth append is fatal only where Postgres holds nothing. An organisation still on a + // dual-write position must not have its run creation failed by the global position. + const store = storeWith({ + modeResolver: resolverOf({ org_dual: "dual-write" }, "redis-only"), + }); + + expect(store.modeForTest("org_dual")).toBe("dual-write"); + expect(store.modeForTest("org_other")).toBe("redis-only"); + expect(store.modeForTest()).toBe("redis-only"); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts index 437872106e7..2892d652c2b 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.off.test.ts @@ -1,6 +1,8 @@ -// Mode off is the merge-test position: the decorator must be indistinguishable from its delegate and -// must not touch Redis at all. A Redis store whose every member throws proves the second half, and -// enumerating the generated name list proves the first for every method rather than a chosen few. +// Mode off with the halt switch thrown is the merge-test position: the decorator must be +// indistinguishable from its delegate and must not touch Redis at all. Plain `off` no longer +// promises that, because a resident run's transitions must keep mirroring; see the hardStop suite. +// A Redis store whose every member throws proves the second half, and enumerating the generated name +// list proves the first for every method rather than a chosen few. import { describe, expect, it } from "vitest"; import { RUN_STORE_METHOD_NAMES } from "./runStoreMethodNames.js"; import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; @@ -46,22 +48,39 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { expect(decorated.mode).toBe("off"); }); - it("forwards every method to the delegate and never calls Redis", async () => { + it("forwards every method to the delegate and never calls Redis when halted", async () => { const { store, calls } = forwardingProbe(); const decorated = new TaskRunExecutionSnapshotStore(store, { store: explodingRedisStore(), mode: "off", + halted: () => true, }) as unknown as Record unknown>; + // Both exceptions return a wrapped handle rather than the delegate's value verbatim, because + // the dial can move at runtime and an unwrapped handle would let a later write bypass the + // decorator with no signal. Every other method is still a pass-through, and the exploding + // Redis store is what proves none of them reaches Redis. + const WRAPPED = ["runInTransaction", "forWaitpointCompletion"]; + for (const name of RUN_STORE_METHOD_NAMES) { - if (name === "runInTransaction") continue; + if (WRAPPED.includes(name)) continue; expect(await decorated[name]("arg-one", "arg-two")).toBe(`result:${name}`); } - expect(calls).toEqual(RUN_STORE_METHOD_NAMES.filter((n) => n !== "runInTransaction")); + const handle = await decorated.forWaitpointCompletion("waitpoint", {}); + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect((handle as unknown as { delegate: unknown }).delegate).toBe( + "result:forWaitpointCompletion" + ); + + // forWaitpointCompletion is called after the loop, so it lands last rather than in place. + expect(calls).toEqual([ + ...RUN_STORE_METHOD_NAMES.filter((n) => !WRAPPED.includes(n)), + "forWaitpointCompletion", + ]); }); - it("hands the delegate's own store to a transaction callback", async () => { + it("wraps the transaction callback's store but never reaches Redis", async () => { const inner = forwardingProbe().store; let seen: unknown; const delegate = { @@ -76,13 +95,18 @@ describe("TaskRunExecutionSnapshotStore at mode off", () => { const decorated = new TaskRunExecutionSnapshotStore(delegate, { store: explodingRedisStore(), mode: "off", + halted: () => true, }); await decorated.runInTransaction("run_1", async (store) => { seen = store; }); - expect(seen).toBe(inner); + // The facade is always installed: this method holds only a runId, so it cannot know whether a + // per-organisation dial would put any write inside on Redis. The exploding Redis store is what + // proves the position still costs nothing at `off`. + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); + expect(seen).not.toBe(inner); }); it("reports every other dial position as one that writes Redis", () => { diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.orgReads.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.orgReads.test.ts new file mode 100644 index 00000000000..7aa4560c8fe --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.orgReads.test.ts @@ -0,0 +1,160 @@ +// Reads are ORG-SCOPED: a single org soaked at redis-read must read from Redis while everyone else +// stays on Postgres, and the dual-write soak phase must pay zero new read cost — no org resolution +// fires until some org is actually read-enabled. Pure predicates plus proxy spies, no containers. +import { describe, expect, it } from "vitest"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { + SnapshotStoreMode, + SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +type CohortProbe = { readsFromRedis(runId: string, environmentId?: string): boolean }; + +function harness(opts: { + globalMode: SnapshotStoreMode; + readModeFor?: (runId: string, environmentId?: string) => SnapshotStoreMode; + anyOrgReadEnabled?: () => boolean; + readPercent?: number; +}) { + const redisTouched: string[] = []; + const redis = new Proxy({} as RedisSnapshotStore, { + get: (_t, prop) => { + return (...__: unknown[]) => { + redisTouched.push(String(prop)); + // findSnapshotCompletedWaitpointIds calls getSnapshotWaitpointIds and returns on present. + return Promise.resolve({ present: true, distinctIds: ["wp_1"], order: [] }); + }; + }, + }); + + const delegateTouched: string[] = []; + const delegate = new Proxy({} as Record, { + get: (_t, prop) => { + return (...__: unknown[]) => { + delegateTouched.push(String(prop)); + return Promise.resolve([]); + }; + }, + }) as unknown as RunStore; + + let readModeForCalls = 0; + const modeResolver: SnapshotStoreModeResolver = { + resolve: () => opts.globalMode, + ...(opts.readModeFor && { + readModeFor: (runId: string, environmentId?: string) => { + readModeForCalls++; + return opts.readModeFor!(runId, environmentId); + }, + }), + ...(opts.anyOrgReadEnabled && { anyOrgReadEnabled: opts.anyOrgReadEnabled }), + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.globalMode, + modeResolver, + readPercent: opts.readPercent ?? 100, + }); + + return { + decorated, + probe: decorated as unknown as CohortProbe, + redisTouched, + delegateTouched, + readModeForCalls: () => readModeForCalls, + }; +} + +describe("org-scoped read routing", () => { + it("serves an enabled org from Redis while the global dial is off", async () => { + // org A at redis-read, global off, some org read-enabled. run_a resolves to org A. + const h = harness({ + globalMode: "off", + anyOrgReadEnabled: () => true, + readModeFor: (runId) => (runId === "run_a" ? "redis-read" : "off"), + }); + + expect(h.probe.readsFromRedis("run_a")).toBe(true); + // Complement to the short-circuit test's ==0: here the dial IS consulted per read. + expect(h.readModeForCalls()).toBeGreaterThan(0); + + await h.decorated.findSnapshotCompletedWaitpointIds("snap_1", undefined, "run_a"); + expect(h.redisTouched).toContain("getSnapshotWaitpointIds"); + }); + + it("keeps a non-enabled org on Postgres and never touches Redis", async () => { + const h = harness({ + globalMode: "off", + anyOrgReadEnabled: () => true, + readModeFor: (runId) => (runId === "run_a" ? "redis-read" : "off"), + }); + + expect(h.probe.readsFromRedis("run_b")).toBe(false); + + await h.decorated.findSnapshotCompletedWaitpointIds("snap_1", undefined, "run_b"); + expect(h.redisTouched).toEqual([]); + expect(h.delegateTouched).toContain("findSnapshotCompletedWaitpointIds"); + }); + + it("short-circuits with no org resolution when no org is read-enabled and the global dial is off", async () => { + const h = harness({ + globalMode: "off", + anyOrgReadEnabled: () => false, + readModeFor: () => "redis-read", + }); + + expect(h.probe.readsFromRedis("run_a")).toBe(false); + // The whole point of the short-circuit: readModeFor is never consulted during the soak. + expect(h.readModeForCalls()).toBe(0); + + await h.decorated.findSnapshotCompletedWaitpointIds("snap_1", undefined, "run_a"); + expect(h.redisTouched).toEqual([]); + expect(h.delegateTouched).toContain("findSnapshotCompletedWaitpointIds"); + }); + + it("threads the environmentId a read site holds into the resolver", () => { + const seen: Array = []; + const h = harness({ + globalMode: "off", + anyOrgReadEnabled: () => true, + readModeFor: (_runId, environmentId) => { + seen.push(environmentId); + return "off"; + }, + }); + + h.probe.readsFromRedis("run_a", "env_9"); + expect(seen).toContain("env_9"); + }); +}); + +describe("resolver is transparent to routing when no per-org override applies", () => { + const ids = Array.from({ length: 500 }, (_, n) => `run_cohort_${n}_${n * 7919}`); + + function plain(mode: SnapshotStoreMode, readPercent: number): CohortProbe { + return new TaskRunExecutionSnapshotStore({} as unknown as RunStore, { + store: {} as never, + mode, + readPercent, + }) as unknown as CohortProbe; + } + + it("routes every run exactly as the global dial did before per-org reads existed", () => { + // A resolver that always answers the global mode, with no org read-enabled. The short-circuit + // does not fire (global IS a read position), effective === global, and the hash path is + // untouched, so the population must route identically to a store with no resolver at all. + const withResolver = harness({ + globalMode: "redis-read", + readPercent: 50, + anyOrgReadEnabled: () => false, + readModeFor: () => "redis-read", + }).probe; + const withoutResolver = plain("redis-read", 50); + + for (const id of ids) { + expect(withResolver.readsFromRedis(id)).toBe(withoutResolver.readsFromRedis(id)); + } + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.rampSites.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.rampSites.test.ts new file mode 100644 index 00000000000..1e2b82a7fda --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.rampSites.test.ts @@ -0,0 +1,181 @@ +// The per-organisation ramp is the canonical procedure: deployment dial down, one organisation opted +// in. Asserted at the CALL SITES, because a predicate seam cannot show a birth and that same run's +// transitions disagreeing, which is the whole failure. A recording append and a sentinel delegate +// answer the only question here, which is who reaches Redis, so no containers are involved; the +// behavioural suites for these sites run against a real Postgres and a real Redis. +import { describe, expect, it } from "vitest"; +import type { RedisSnapshotStore, SnapshotEntryInput } from "./redisSnapshotStore.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { + SnapshotStoreMode, + SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +const ORG = "org_ramp"; +const RUN = "run_ramp"; + +const SCOPE = { + environmentId: "env_1", + environmentType: "PRODUCTION", + projectId: "proj_1", + organizationId: ORG, +} as const; + +function harness(global: SnapshotStoreMode, perOrg: SnapshotStoreMode) { + const appends: { kind: string; entry: SnapshotEntryInput }[] = []; + + const redis = { + append: async (args: { entry: SnapshotEntryInput; kind: string }) => { + appends.push({ kind: args.kind, entry: args.entry }); + return { outcome: "written" as const, seq: appends.length }; + }, + } as unknown as RedisSnapshotStore; + + const delegate = new Proxy({} as Record, { + get: (_target, prop: string) => () => + Promise.resolve(prop === "expireParkedRun" ? { count: 1 } : {}), + }) as unknown as RunStore; + + const modeResolver = { + resolve: (organizationId?: string) => (organizationId === ORG ? perOrg : global), + } satisfies SnapshotStoreModeResolver; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: global, + modeResolver, + readPercent: 100, + }); + + return { decorated, appends }; +} + +async function births(store: TaskRunExecutionSnapshotStore): Promise { + await store.createRun({ + data: { id: RUN } as never, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run created", + runStatus: "PENDING", + ...SCOPE, + }, + }); + + await store.createCancelledRun({ + data: { id: `${RUN}_cancelled` } as never, + snapshot: { + engine: "V2", + executionStatus: "FINISHED", + description: "Run was cancelled", + runStatus: "CANCELED", + ...SCOPE, + }, + }); +} + +async function transitions(store: TaskRunExecutionSnapshotStore): Promise { + await store.completeAttemptSuccess( + RUN, + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + executionStatus: "FINISHED", + description: "Attempt succeeded", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + ...SCOPE, + }, + }, + { select: { id: true } } + ); + + const expireSnapshot = { + engine: "V2", + executionStatus: "FINISHED", + description: "Run expired", + runStatus: "EXPIRED", + ...SCOPE, + } as const; + + await store.expireRun( + RUN, + { error: {}, completedAt: new Date(), expiredAt: new Date(), snapshot: expireSnapshot }, + { select: { id: true } } + ); + + await store.expireParkedRun(RUN, { + error: {}, + completedAt: new Date(), + expiredAt: new Date(), + statusReason: "expired", + snapshot: expireSnapshot, + }); + + await store.rescheduleRun(RUN, { delayUntil: new Date(), snapshot: { ...SCOPE } }); + + await store.lockRunToWorker(RUN, { + lockedAt: new Date(), + lockedById: "worker_1", + lockedToVersionId: "version_1", + lockedQueueId: "queue_1", + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + sdkVersion: null, + cliVersion: null, + maxDurationInSeconds: null, + snapshot: { + id: "snap_lock", + previousSnapshotId: "snap_previous", + completedWaitpointIds: [], + completedWaitpointOrder: [], + ...SCOPE, + }, + }); + + await store.createExecutionSnapshot({ + run: { id: RUN, status: "EXECUTING" }, + snapshot: { executionStatus: "EXECUTING", description: "Executing" }, + ...SCOPE, + }); +} + +const TRANSITION_COUNT = 6; + +describe("the per-organisation ramp", () => { + it("mirrors a birth AND that run's transitions for an organisation opted in past a dial at off", async () => { + const { decorated, appends } = harness("off", "dual-write"); + + await births(decorated); + expect(appends.map((a) => a.kind)).toEqual(["birth", "birth"]); + + appends.length = 0; + await transitions(decorated); + expect(appends).toHaveLength(TRANSITION_COUNT); + expect(appends.every((a) => a.kind === "transition")).toBe(true); + }); + + // Named for what it can prove. It asserts which CALL SITES reach Redis, which is a seam-level + // property: the append here is a recording double, so no keyspace exists and residency itself is + // not exercised. That a RESIDENT run keeps mirroring after its organisation moves to off is + // covered against real infrastructure elsewhere, and was verified by hand as well: a run born at + // dual-write went from 3 entries to 8 with its head matching Postgres after the organisation was + // pinned off mid-flight. + it("asks Redis at no birth site for an organisation held at off, and still asks at every transition site", async () => { + const { decorated, appends } = harness("dual-write", "off"); + + await births(decorated); + expect(appends).toHaveLength(0); + + // Residency is the keyspace, and the append script refuses a transition into one that does not + // exist. Asking the organisation again here is what lets a run change stores mid-life. + await transitions(decorated); + expect(appends).toHaveLength(TRANSITION_COUNT); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnlyFallback.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnlyFallback.test.ts new file mode 100644 index 00000000000..77facf97b2c --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.redisOnlyFallback.test.ts @@ -0,0 +1,443 @@ +// At `redis-only` Postgres holds no snapshots, so a Redis ERROR must NOT fall back to an empty +// Postgres — it must throw (retryable) so the run is not stranded. The gate is org-aware: a run whose +// org resolves to `redis-only` throws; an unresolved org throws only when some org is `redis-only` +// (conservative over-throw); a run resolved to a non-`redis-only` mode always falls back, even when a +// DIFFERENT org is `redis-only`. Pure predicates plus proxy spies, no containers. +import { describe, expect, it } from "vitest"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { + SnapshotStoreMode, + SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +function harness(opts: { + globalMode: SnapshotStoreMode; + readModeFor?: (runId: string, environmentId?: string) => SnapshotStoreMode | undefined; + anyOrgRedisOnly?: () => boolean; + anyOrgReadEnabled?: () => boolean; +}) { + // Every Redis call throws: this is the brownout the fallback gate exists for. + const redis = new Proxy({} as RedisSnapshotStore, { + get: () => () => { + throw new Error("redis brownout"); + }, + }); + + const delegateTouched: string[] = []; + const delegate = new Proxy({} as Record, { + get: (_t, prop) => { + return (...__: unknown[]) => { + delegateTouched.push(String(prop)); + return Promise.resolve({ id: "pg_fallback" }); + }; + }, + }) as unknown as RunStore; + + const modeResolver: SnapshotStoreModeResolver = { + resolve: () => opts.globalMode, + ...(opts.readModeFor && { + readModeFor: opts.readModeFor as SnapshotStoreModeResolver["readModeFor"], + }), + ...(opts.anyOrgRedisOnly && { anyOrgRedisOnly: opts.anyOrgRedisOnly }), + ...(opts.anyOrgReadEnabled && { anyOrgReadEnabled: opts.anyOrgReadEnabled }), + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.globalMode, + modeResolver, + readPercent: 100, + }); + + return { decorated, delegateTouched }; +} + +describe("redis-only fallback gate is org-aware", () => { + it("(a) throws for a run whose org resolves to redis-only, never serving Postgres", async () => { + const h = harness({ + globalMode: "dual-write", + anyOrgReadEnabled: () => true, + readModeFor: (runId) => (runId === "run_a" ? "redis-only" : undefined), + }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_a", undefined, "env_a") + ).rejects.toThrow("redis brownout"); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("(b) throws for an unresolved org while some org is redis-only", async () => { + const h = harness({ + globalMode: "redis-read", + readModeFor: () => undefined, + anyOrgRedisOnly: () => true, + }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_x") + ).rejects.toThrow("redis brownout"); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("(c) falls back for an unresolved org when no org is redis-only", async () => { + const h = harness({ + globalMode: "redis-read", + readModeFor: () => undefined, + anyOrgRedisOnly: () => false, + }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_x"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); + + it("(d) falls back for a run resolved to redis-read even when another org is redis-only", async () => { + const h = harness({ + globalMode: "redis-read", + readModeFor: () => "redis-read", + anyOrgRedisOnly: () => true, + }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_x"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); + + it("(e) throws when the global dial is redis-only (unchanged)", async () => { + const h = harness({ globalMode: "redis-only" }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_x") + ).rejects.toThrow("redis brownout"); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("threads runId into a runId-only read site (findSnapshotCompletedWaitpointIds)", async () => { + const seen: string[] = []; + const h = harness({ + globalMode: "redis-read", + anyOrgRedisOnly: () => true, + readModeFor: (runId) => { + seen.push(runId); + return runId === "run_a" ? "redis-only" : undefined; + }, + }); + + await expect( + h.decorated.findSnapshotCompletedWaitpointIds("snap_1", undefined, "run_a") + ).rejects.toThrow("redis brownout"); + expect(seen).toContain("run_a"); + expect(h.delegateTouched).not.toContain("findSnapshotCompletedWaitpointIds"); + }); +}); + +// The other half of the gate: a Redis MISS (not an error). At redis-only Postgres holds nothing, +// so a miss must THROW rather than delegate into an empty Postgres. Below redis-only a miss still +// falls back, because Postgres legitimately holds the pre-cutover data. +function missHarness(opts: { + globalMode: SnapshotStoreMode; + readModeFor?: (runId: string, environmentId?: string) => SnapshotStoreMode | undefined; + anyOrgRedisOnly?: () => boolean; + anyOrgReadEnabled?: () => boolean; +}) { + // Every Redis read reports a MISS: null for the single/by-id reads, a miss window, an absent + // waitpoint set. + const redis = new Proxy({} as RedisSnapshotStore, { + get: (_t, prop) => { + if (prop === "getSinceCreatedAt") return () => Promise.resolve({ kind: "miss" }); + if (prop === "getSnapshotWaitpointIds") + return () => Promise.resolve({ present: false, distinctIds: [], order: [] }); + return () => Promise.resolve(null); + }, + }); + + const delegateTouched: string[] = []; + const delegate = new Proxy({} as Record, { + get: (_t, prop) => { + return (...__: unknown[]) => { + delegateTouched.push(String(prop)); + return Promise.resolve({ id: "pg_fallback" }); + }; + }, + }) as unknown as RunStore; + + const modeResolver: SnapshotStoreModeResolver = { + resolve: () => opts.globalMode, + ...(opts.readModeFor && { + readModeFor: opts.readModeFor as SnapshotStoreModeResolver["readModeFor"], + }), + ...(opts.anyOrgRedisOnly && { anyOrgRedisOnly: opts.anyOrgRedisOnly }), + ...(opts.anyOrgReadEnabled && { anyOrgReadEnabled: opts.anyOrgReadEnabled }), + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.globalMode, + modeResolver, + readPercent: 100, + }); + + return { decorated, delegateTouched }; +} + +describe("redis-only throws on a miss instead of serving empty Postgres", () => { + it("findLatestExecutionSnapshot throws at global redis-only, never delegating", async () => { + const h = missHarness({ globalMode: "redis-only" }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_x") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("findLatestExecutionSnapshot falls back at redis-read (Postgres holds pre-cutover data)", async () => { + const h = missHarness({ globalMode: "redis-read" }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_x"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); + + it("throws for a run whose org resolves to redis-only, falls back for a redis-read org", async () => { + const h = missHarness({ + globalMode: "redis-read", + anyOrgReadEnabled: () => true, + readModeFor: (runId) => (runId === "run_ro" ? "redis-only" : "redis-read"), + }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_ro", undefined, "env_a") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + + const result = await h.decorated.findLatestExecutionSnapshot("run_rr", undefined, "env_b"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); + + it("findManyExecutionSnapshots throws on a miss window at redis-only", async () => { + const h = missHarness({ globalMode: "redis-only" }); + + await expect( + h.decorated.findManyExecutionSnapshots({ + where: { runId: "run_x", isValid: true, createdAt: { gt: new Date(0) } }, + include: { checkpoint: true }, + orderBy: { createdAt: "desc" }, + take: 10, + } as never) + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findManyExecutionSnapshots"); + }); + + it("findSnapshotCompletedWaitpointIds throws on an absent set at redis-only", async () => { + const h = missHarness({ globalMode: "redis-only" }); + + await expect( + h.decorated.findSnapshotCompletedWaitpointIds("snap_1", undefined, "run_x") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findSnapshotCompletedWaitpointIds"); + }); + + it("findExecutionSnapshot throws on a by-id miss at redis-only", async () => { + const h = missHarness({ globalMode: "redis-only" }); + + await expect( + h.decorated.findExecutionSnapshot({ + where: { id: "snap_1", runId: "run_x" }, + select: { createdAt: true }, + } as never) + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findExecutionSnapshot"); + }); +}); + +// The unified decision: when the run→org cache is COLD (readModeFor undefined) and some org is +// redis-only, the sync gate cannot tell whether THIS run is redis-only. It must resolve the run's +// org authoritatively rather than either strand a redis-only run (empty Postgres) or over-throw a +// pre-cutover run. This harness supplies that authoritative hook and a configurable Redis read. +function authHarness(opts: { + globalMode: SnapshotStoreMode; + read: "miss" | "dangling" | "error"; + readModeFor?: (runId: string, environmentId?: string) => SnapshotStoreMode | undefined; + anyOrgRedisOnly?: () => boolean; + anyOrgReadEnabled?: () => boolean; + authoritative?: (runId: string) => Promise; +}) { + const danglingRead = { + id: "snap_head", + seq: 1, + isValid: true, + entry: { id: "snap_head", createdAt: new Date().toISOString() }, + raw: "{}", + danglingCycle: true, + }; + const redisRead = () => { + if (opts.read === "error") throw new Error("redis brownout"); + return Promise.resolve(opts.read === "dangling" ? danglingRead : null); + }; + const redis = new Proxy({} as RedisSnapshotStore, { + get: (_t, prop) => { + if (prop === "getSinceCreatedAt") + return () => (opts.read === "error" ? redisRead() : Promise.resolve({ kind: "miss" })); + if (prop === "getSnapshotWaitpointIds") + return () => + opts.read === "error" + ? redisRead() + : Promise.resolve({ present: false, distinctIds: [], order: [] }); + return () => redisRead(); + }, + }); + + const delegateTouched: string[] = []; + const delegate = new Proxy({} as Record, { + get: (_t, prop) => { + return (...__: unknown[]) => { + delegateTouched.push(String(prop)); + return Promise.resolve({ id: "pg_fallback" }); + }; + }, + }) as unknown as RunStore; + + const authoritativeCalls: string[] = []; + const modeResolver: SnapshotStoreModeResolver = { + resolve: () => opts.globalMode, + ...(opts.readModeFor && { + readModeFor: opts.readModeFor as SnapshotStoreModeResolver["readModeFor"], + }), + ...(opts.anyOrgRedisOnly && { anyOrgRedisOnly: opts.anyOrgRedisOnly }), + ...(opts.anyOrgReadEnabled && { anyOrgReadEnabled: opts.anyOrgReadEnabled }), + ...(opts.authoritative && { + readModeForAuthoritative: (async (runId: string) => { + authoritativeCalls.push(runId); + return opts.authoritative!(runId); + }) as SnapshotStoreModeResolver["readModeForAuthoritative"], + }), + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.globalMode, + modeResolver, + readPercent: 100, + }); + + return { decorated, delegateTouched, authoritativeCalls }; +} + +describe("redis-only fallback resolves a cold run→org cache authoritatively", () => { + it("routing branch: a cold-cache redis-only run at global dual-write THROWS, never empty PG", async () => { + const h = authHarness({ + globalMode: "dual-write", + read: "miss", + readModeFor: () => undefined, + anyOrgRedisOnly: () => true, + authoritative: async () => "redis-only", + }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_cold", undefined, "env_a") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + expect(h.authoritativeCalls).toContain("run_cold"); + }); + + it("routing branch: a cold-cache dual-write run falls back (no over-throw)", async () => { + const h = authHarness({ + globalMode: "dual-write", + read: "miss", + readModeFor: () => undefined, + anyOrgRedisOnly: () => true, + authoritative: async () => "dual-write", + }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_cold", undefined, "env_a"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); + + it("off/dual-write with no org redis-only falls back WITHOUT an authoritative read", async () => { + const h = authHarness({ + globalMode: "dual-write", + read: "miss", + readModeFor: () => undefined, + anyOrgRedisOnly: () => false, + authoritative: async () => "redis-only", + }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_a"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.authoritativeCalls).toEqual([]); + }); + + it("MISS at global redis-read with a cold-cache pre-cutover run FALLS BACK (coexistence)", async () => { + const h = authHarness({ + globalMode: "redis-read", + read: "miss", + anyOrgReadEnabled: () => true, + readModeFor: () => undefined, + anyOrgRedisOnly: () => true, + authoritative: async () => "redis-read", + }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_pre", undefined, "env_a"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); + + it("MISS at global redis-read with a cold-cache redis-only run THROWS", async () => { + const h = authHarness({ + globalMode: "redis-read", + read: "miss", + anyOrgReadEnabled: () => true, + readModeFor: () => undefined, + anyOrgRedisOnly: () => true, + authoritative: async () => "redis-only", + }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_ro", undefined, "env_a") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("fails closed (THROWS) when the authoritative read itself fails and some org is redis-only", async () => { + const h = authHarness({ + globalMode: "redis-read", + read: "miss", + anyOrgReadEnabled: () => true, + readModeFor: () => undefined, + anyOrgRedisOnly: () => true, + authoritative: async () => { + throw new Error("run→org read timed out"); + }, + }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_cold", undefined, "env_a") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("danglingCycle at global redis-only THROWS, never empty PG", async () => { + const h = authHarness({ globalMode: "redis-only", read: "dangling" }); + + await expect( + h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_a") + ).rejects.toThrow(/redis-only/); + expect(h.delegateTouched).not.toContain("findLatestExecutionSnapshot"); + }); + + it("danglingCycle below redis-only still falls back to Postgres", async () => { + const h = authHarness({ + globalMode: "redis-read", + read: "dangling", + anyOrgReadEnabled: () => true, + }); + + const result = await h.decorated.findLatestExecutionSnapshot("run_x", undefined, "env_a"); + expect(result).toEqual({ id: "pg_fallback" }); + expect(h.delegateTouched).toContain("findLatestExecutionSnapshot"); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.repair.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.repair.test.ts new file mode 100644 index 00000000000..bcad0025426 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.repair.test.ts @@ -0,0 +1,380 @@ +// A lost Redis append leaves the mirrored head stale while Postgres moves on. The repair job's whole +// job is to close that gap, and it can only do so by re-appending the POSTGRES head: reading through +// the decorator would serve Redis's own stale head back to it and it would conclude there is nothing +// to repair. +// +// Container-free on purpose: every behaviour asserted here is a decision the decorator makes about +// which store it reads and what it hands the append script. The append script's own idempotency and +// its no-keyspace refusal are proved against a real Redis in redisSnapshotStore.test.ts, and this +// file asserts that the repair routes into those two outcomes rather than around them. +import { describe, expect, it } from "vitest"; +import type { AppendResult, RedisSnapshotStore, SnapshotRead } from "./redisSnapshotStore.js"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +/** The seven statuses the decorator's transition path can lose an append for. */ +const TRANSITION_STATUSES = [ + "EXECUTING", + "EXECUTING_WITH_WAITPOINTS", + "PENDING_CANCEL", + "PENDING_EXECUTING", + "QUEUED_EXECUTING", + "RUN_CREATED", + "DELAYED", +] as const; + +type PgRow = Awaited>; + +function pgHead(overrides: { + id: string; + runId: string; + executionStatus: string; + createdAt: Date; + previousSnapshotId?: string | null; + completedWaitpointOrder?: string[]; + completedWaitpoints?: { id: string }[]; +}): PgRow { + return { + id: overrides.id, + engine: "V2", + executionStatus: overrides.executionStatus, + description: "repair fixture", + isValid: true, + error: null, + previousSnapshotId: overrides.previousSnapshotId ?? null, + runId: overrides.runId, + runStatus: "EXECUTING", + batchId: null, + attemptNumber: 1, + environmentId: "env_1", + environmentType: "PRODUCTION", + projectId: "proj_1", + organizationId: "org_1", + completedWaitpoints: overrides.completedWaitpoints ?? [], + completedWaitpointOrder: overrides.completedWaitpointOrder ?? [], + checkpointId: null, + checkpoint: null, + workerId: null, + runnerId: null, + createdAt: overrides.createdAt, + updatedAt: overrides.createdAt, + lastHeartbeatAt: null, + metadata: null, + } as unknown as PgRow; +} + +type Appended = Parameters[0]; + +/** + * Stands in for the append script. It answers with whichever outcome the scenario is about and + * records what it was handed, which is the only way to assert the repair never asks for a delete or + * an expiry. + */ +class RecordingRedis { + readonly appends: Appended[] = []; + readonly calls: string[] = []; + + constructor( + private head: SnapshotRead | null, + private outcome: AppendResult = { + outcome: "written", + seq: 9, + ttl: "none", + cycleMismatch: false, + } + ) {} + + async getLatest(_runId: string): Promise { + this.calls.push("getLatest"); + return this.head; + } + + async append(args: Appended): Promise { + this.calls.push("append"); + this.appends.push(args); + return this.outcome; + } + + async dropRun(): Promise { + this.calls.push("dropRun"); + } + + gapsMarked = 0; + + async markGapsIfResident(): Promise { + this.calls.push("markGapsIfResident"); + this.gapsMarked += 1; + return true; + } +} + +function redisHead(id: string, createdAt: Date): SnapshotRead { + return { + id, + seq: 4, + isValid: true, + entry: { id, createdAt: createdAt.toISOString() }, + raw: "{}", + }; +} + +/** + * Only the two collaborators the repair reads through are supplied. `findLatestExecutionSnapshot` on + * the delegate IS the Postgres truth; a repair that reaches for the decorated view instead sees the + * stale mirror. + */ +function harness(opts: { + pg: PgRow; + redis: RecordingRedis; + mode?: SnapshotStoreMode; + onDelegateRead?: () => void; +}) { + const delegate = { + findLatestExecutionSnapshot: async () => { + opts.onDelegateRead?.(); + return opts.pg; + }, + } as unknown as RunStore; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: opts.redis as unknown as RedisSnapshotStore, + mode: opts.mode ?? "redis-read", + }); + + return decorated; +} + +describe("repairRedisHead", () => { + it.each(TRANSITION_STATUSES)( + "re-appends the Postgres head for a lost %s append", + async (executionStatus) => { + const runId = "run_1"; + const lost = "snap_lost"; + const created = new Date("2026-01-01T00:00:10.000Z"); + const redis = new RecordingRedis( + redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z")) + ); + + const store = harness({ + pg: pgHead({ id: lost, runId, executionStatus, createdAt: created }), + redis, + }); + + await expect(store.repairRedisHead(runId, lost)).resolves.toBe("reappended"); + + expect(redis.appends).toHaveLength(1); + expect(redis.appends[0]!.kind).toBe("transition"); + expect(redis.appends[0]!.entry.id).toBe(lost); + expect(redis.appends[0]!.entry.executionStatus).toBe(executionStatus); + expect(redis.appends[0]!.entry.createdAt).toBe(created.toISOString()); + } + ); + + it("never asks the store to delete or expire anything", async () => { + const redis = new RecordingRedis(redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + }); + + await store.repairRedisHead("run_1", "snap_lost"); + + expect(redis.calls).not.toContain("dropRun"); + }); + + it("appends without a compare-and-set, so a gap wider than one entry still restores the head", async () => { + // Asserting cur would make the repair fail exactly when Redis is furthest behind, which is when + // the stale head is doing the most damage. + const redis = new RecordingRedis( + redisHead("snap_two_back", new Date("2026-01-01T00:00:00.000Z")) + ); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + previousSnapshotId: "snap_one_back", + }), + redis, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("reappended"); + expect(redis.appends[0]!.expectedCur).toBeUndefined(); + }); + + it("reads the Postgres head rather than the mirrored view", async () => { + let delegateReads = 0; + // The mirror's head is a DIFFERENT, older snapshot. A repair reading the mirror would compare + // that id against the one it was asked to repair, find no match, and abort. + const redis = new RecordingRedis(redisHead("snap_stale", new Date("2026-01-01T00:00:00.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + onDelegateRead: () => { + delegateReads += 1; + }, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("reappended"); + expect(delegateReads).toBe(1); + }); + + it("carries the head's completed waitpoints so the repaired entry keeps its wait cycle", async () => { + const redis = new RecordingRedis(redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING_WITH_WAITPOINTS", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + completedWaitpoints: [{ id: "wp_a" }, { id: "wp_b" }], + completedWaitpointOrder: ["wp_a"], + }), + redis, + }); + + await store.repairRedisHead("run_1", "snap_lost"); + + expect(redis.appends[0]!.cycle?.kind).toBe("new"); + expect(redis.appends[0]!.cycle?.completedWaitpoints).toEqual([ + { id: "wp_a", index: 0 }, + { id: "wp_b" }, + ]); + }); + + it("reports a duplicate instead of treating an already-landed entry as a failure", async () => { + const redis = new RecordingRedis(redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z")), { + outcome: "duplicate", + seq: 5, + }); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("duplicate"); + }); + + it("does not resurrect a run that was never resident in Redis", async () => { + // No keyspace is the record of non-residency. The append script refuses the transition and the + // repair must report that, not retry it into existence some other way. + const redis = new RecordingRedis(null, { outcome: "skippedNoKeyspace" }); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("notResident"); + expect(redis.appends[0]!.kind).toBe("transition"); + }); + + it("appends nothing when the mirror already holds the Postgres head, but still marks the hole", async () => { + const redis = new RecordingRedis(redisHead("snap_lost", new Date("2026-01-01T00:00:10.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("alreadyCurrent"); + expect(redis.appends).toHaveLength(0); + // The head converged on its own, but the repair only ran because an append was lost, so entries + // behind it can still be missing. Observed live: four entries in Redis against eight in + // Postgres, with a matching head. Without the mark, that keyspace serves short windows as whole. + expect(redis.gapsMarked).toBe(1); + }); + + it("refuses to append behind a newer mirror head, and marks the hole", async () => { + // Appending an older entry at the tail would leave the chain claiming a state the run has left. + const redis = new RecordingRedis(redisHead("snap_newer", new Date("2026-01-01T00:00:20.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("redisAhead"); + expect(redis.appends).toHaveLength(0); + // Divergence either way round is still divergence. + expect(redis.gapsMarked).toBe(1); + }); + + it("heals the head even when the run has transitioned past the lost snapshot", async () => { + // The repair is delayed by a minute, so the run routinely moves on before it runs. The entry + // that was lost is unreachable by then, but the mirror head is still wrong, and a wrong head is + // what a Redis-served read returns. So the target is whatever Postgres holds now. + const redis = new RecordingRedis(redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_later", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("reappended"); + expect(redis.appends[0]!.entry.id).toBe("snap_later"); + }); + + it("reports that there is nothing to repair when Postgres holds no snapshot", async () => { + const redis = new RecordingRedis(redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z"))); + const store = harness({ pg: null, redis }); + + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("noSnapshot"); + expect(redis.calls).toHaveLength(0); + }); + + it("still heals a resident run when the deployment-wide dial is off", async () => { + const redis = new RecordingRedis(redisHead("snap_prev", new Date("2026-01-01T00:00:00.000Z"))); + const store = harness({ + pg: pgHead({ + id: "snap_lost", + runId: "run_1", + executionStatus: "EXECUTING", + createdAt: new Date("2026-01-01T00:00:10.000Z"), + }), + redis, + mode: "off", + }); + + // The dial governs births. A run that is already resident keeps its mirror, and healing it is + // the whole point of the repair: refusing here would leave a head frozen for the rest of the + // run's life precisely because an operator lowered the dial. + await expect(store.repairRedisHead("run_1", "snap_lost")).resolves.toBe("reappended"); + expect(redis.calls).toContain("append"); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.repairEndToEnd.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.repairEndToEnd.test.ts new file mode 100644 index 00000000000..bb86ffa1378 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.repairEndToEnd.test.ts @@ -0,0 +1,227 @@ +// The whole loss-and-recovery cycle against a real Postgres and a real Redis: an injected fault +// models the process dying between the two writes, and the repair is then asked to close the gap it +// left. Only the append script can prove the two guards the repair leans on, so the sibling +// container-free suite covers the decisions and this one covers the guards. +import { describe, expect } from "vitest"; +import { containerTest } from "@internal/testcontainers"; +import { generateInternalId } from "@trigger.dev/core/v3/isomorphic"; +import { PostgresRunStore } from "./PostgresRunStore.js"; +import { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import { entryFromCreateRun } from "./snapshotEntry.js"; +import { InjectedSnapshotFault } from "./snapshotFaultInjection.js"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; +import { + buildCreateRunData, + seedSnapshotEnvironment, + type SnapshotFixtureEnv, +} from "./testFixtures/snapshotIdFixture.js"; + +const COMPLETED_TTL_MS = 72 * 60 * 60 * 1000; + +function harness( + prisma: never, + redisOptions: never, + opts?: { faults?: ConstructorParameters[1]["faults"] } +) { + const redis = new RedisSnapshotStore({ redisOptions, completedTtlMs: COMPLETED_TTL_MS }); + const repairs: { runId: string; snapshotId: string; executionStatus: string }[] = []; + + const decorated = new TaskRunExecutionSnapshotStore( + new PostgresRunStore({ prisma, readOnlyPrisma: prisma }) as unknown as RunStore, + { + store: redis, + mode: "redis-read", + readPercent: 100, + ...(opts?.faults && { faults: opts.faults }), + onAppendFailure: async (args) => { + repairs.push(args); + }, + } + ); + + return { decorated, redis, repairs }; +} + +async function seedBirth( + decorated: TaskRunExecutionSnapshotStore, + redis: RedisSnapshotStore, + runId: string, + env: SnapshotFixtureEnv +): Promise { + const snapshot = { + id: generateInternalId(), + engine: "V2" as const, + executionStatus: "RUN_CREATED" as const, + description: "Run was created", + runStatus: "PENDING" as const, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }; + + await redis.append({ + entry: entryFromCreateRun({ id: snapshot.id, runId, createdAt: new Date() }, snapshot), + kind: "birth", + isTerminal: false, + }); + + await decorated.createRun({ data: buildCreateRunData(runId, env), snapshot }); +} + +function executingSnapshot(runId: string, env: SnapshotFixtureEnv, previousSnapshotId?: string) { + return { + id: generateInternalId(), + run: { id: runId, status: "EXECUTING" as const, attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING" as const, description: "Run is executing" }, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + ...(previousSnapshotId && { previousSnapshotId }), + }; +} + +describe("snapshot repair end to end", () => { + containerTest( + "re-appends an EXECUTING snapshot whose append the process died before making", + async ({ prisma, redisOptions }) => { + let dropNext = true; + const { decorated, redis, repairs } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis" && dropNext) { + dropNext = false; + throw new InjectedSnapshotFault(boundary); + } + }, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot(executingSnapshot(runId, env)); + + expect(repairs).toEqual([{ runId, snapshotId: created.id, executionStatus: "EXECUTING" }]); + expect(await redis.getById(runId, created.id)).toBeNull(); + + await expect(decorated.repairRedisHead(runId, created.id)).resolves.toBe("reappended"); + + const read = await redis.getLatest(runId); + expect(read!.id).toBe(created.id); + expect(read!.entry["executionStatus"]).toBe("EXECUTING"); + expect(read!.entry["createdAt"]).toBe(created.createdAt.toISOString()); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "is safe to run twice: the second attempt adds no second entry", + async ({ prisma, redisOptions }) => { + let dropNext = true; + const { decorated, redis } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis" && dropNext) { + dropNext = false; + throw new InjectedSnapshotFault(boundary); + } + }, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const created = await decorated.createExecutionSnapshot(executingSnapshot(runId, env)); + + await expect(decorated.repairRedisHead(runId, created.id)).resolves.toBe("reappended"); + const first = await redis.getLatest(runId); + + await expect(decorated.repairRedisHead(runId, created.id)).resolves.toBe("alreadyCurrent"); + const second = await redis.getLatest(runId); + + expect(second!.seq).toBe(first!.seq); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "does not resurrect a run that was never resident in Redis", + async ({ prisma, redisOptions }) => { + const { decorated, redis } = harness(prisma as never, redisOptions as never); + + try { + // Born through the UNDECORATED store, so Postgres holds a head and Redis holds no keyspace. + // This is every pre-cutover run. + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + const plain = new PostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }) as unknown as RunStore; + + await plain.createRun({ + data: buildCreateRunData(runId, env), + snapshot: { + id: generateInternalId(), + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + }, + }); + + const head = await plain.findLatestExecutionSnapshot(runId); + + await expect(decorated.repairRedisHead(runId, head!.id)).resolves.toBe("notResident"); + expect(await redis.getLatest(runId)).toBeNull(); + } finally { + await redis.quit(); + } + } + ); + + containerTest( + "heals the head after the run has transitioned past the lost snapshot", + async ({ prisma, redisOptions }) => { + let dropping = true; + const { decorated, redis } = harness(prisma as never, redisOptions as never, { + faults: (boundary) => { + if (boundary === "afterPgBeforeRedis" && dropping) { + throw new InjectedSnapshotFault(boundary); + } + }, + }); + + try { + const env = await seedSnapshotEnvironment(prisma); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); + + const lost = await decorated.createExecutionSnapshot(executingSnapshot(runId, env)); + const later = await decorated.createExecutionSnapshot( + executingSnapshot(runId, env, lost.id) + ); + dropping = false; + + await expect(decorated.repairRedisHead(runId, lost.id)).resolves.toBe("reappended"); + + const read = await redis.getLatest(runId); + expect(read!.id).toBe(later.id); + } finally { + await redis.quit(); + } + } + ); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.residency.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.residency.test.ts new file mode 100644 index 00000000000..7e3e09a1661 --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.residency.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { TaskRunExecutionSnapshotStore } from "./taskRunExecutionSnapshotStore.js"; +import type { + SnapshotStoreMode, + SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +// A run's store is chosen once, at birth, and never changes for that run's life. The dial may move +// underneath it, and the per-organisation override may flip, and neither may divert a run that is +// already running: that is a mid-life store switch, and it leaves the Redis head frozen while +// Postgres advances. +function resolverOf(perOrg: Record, global: SnapshotStoreMode) { + return { + resolve: (organizationId?: string) => (organizationId && perOrg[organizationId]) || global, + } satisfies SnapshotStoreModeResolver; +} + +function storeWith(mode: SnapshotStoreMode, resolver?: SnapshotStoreModeResolver) { + return new TaskRunExecutionSnapshotStore({} as unknown as RunStore, { + store: {} as never, + mode, + ...(resolver && { modeResolver: resolver }), + }); +} + +describe("store residency is decided at birth", () => { + it("lets the per-organisation override decide a BIRTH", () => { + const s = storeWith("off", resolverOf({ org_off: "off" }, "dual-write")); + expect(s.writesRedisForBirthTest("org_off")).toBe(false); + expect(s.writesRedisForBirthTest("org_other")).toBe(true); + }); + + it("does NOT let the dial decide a TRANSITION, at either scope", () => { + // The run is already resident or already absent. Asking the dial again is what allows a run to + // change stores half way through its life. The seam takes an organisation id but must ignore it + // for a transition. Stopping writes outright is the halt switch; see the hardStop suite. + expect( + storeWith("off", resolverOf({ org_off: "off" }, "dual-write")).writesRedisForTransitionTest() + ).toBe(true); + expect( + storeWith("off", resolverOf({ org_on: "dual-write" }, "off")).writesRedisForTransitionTest() + ).toBe(true); + }); + + it("keeps transitions on for a resident run at every position past off", () => { + for (const m of ["dual-write", "redis-read", "redis-only"] as const) { + const s = storeWith("off", resolverOf({ org_off: "off" }, m)); + expect(s.writesRedisForTransitionTest()).toBe(true); + } + }); + + it("keeps an org-scoped transition on once the global dial has ever moved, whatever the census says", () => { + // The sound skip needs BOTH the global dial unmoved AND the org definitely-never-enabled. With + // the global latch set, a resident run exists, so an org-scoped transition must still mirror even + // if the census believes the org was never enabled. + const s = new TaskRunExecutionSnapshotStore({} as unknown as RunStore, { + store: {} as never, + mode: "off", + modeResolver: { + resolve: () => "off", + globalModeEverEnabled: () => true, + orgDefinitelyNeverEnabled: () => true, + }, + }); + expect(s.writesRedisForTransitionTest("org_a")).toBe(true); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts index 9952f569e2b..0c86aad7d14 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.staging.test.ts @@ -216,7 +216,7 @@ describe("the staging facade", () => { ); containerTest( - "hands the transaction callback the plain delegate at mode off", + "stages nothing and touches no Redis inside a transaction at mode off", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); try { @@ -242,7 +242,10 @@ describe("the staging facade", () => { seen = store; }); - expect(seen).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + // The callback now always receives the staging facade: this method holds only a runId, and + // a per-organisation dial lives in that organisation's blob, so it cannot know whether any + // write inside will reach Redis. What must hold at `off` is that nothing is appended. + expect(seen).toBeInstanceOf(TaskRunExecutionSnapshotStore); expect(await redis.getLatest(runId)).toBeNull(); } finally { await redis.quit(); @@ -269,7 +272,7 @@ describe("the staging facade", () => { ); containerTest( - "returns the plain handle from forWaitpointCompletion at mode off", + "wraps the forWaitpointCompletion handle at mode off too", async ({ prisma, redisOptions }) => { const { decorated, redis } = build(prisma as never, redisOptions as never, "off"); try { @@ -277,7 +280,9 @@ describe("the staging facade", () => { routeKind: "MANUAL", } as never); - expect(handle).not.toBeInstanceOf(TaskRunExecutionSnapshotStore); + // Leaving it unwrapped was the one hole a future snapshot write could slip through with no + // signal, and the dial can now move at runtime, so the handle is wrapped at every position. + expect(handle).toBeInstanceOf(TaskRunExecutionSnapshotStore); } finally { await redis.quit(); } diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitionFatality.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitionFatality.test.ts new file mode 100644 index 00000000000..336e3bf51ec --- /dev/null +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitionFatality.test.ts @@ -0,0 +1,162 @@ +// A lost transition append is survivable below redis-only: Postgres already committed and holds the +// head, so the decorator records the failure, enqueues a Postgres-based repair, and returns. At +// redis-only Postgres holds no snapshot, so a lost transition is unrecoverable — the repair cannot +// help — and the append must THROW so the caller sees the loss, mirroring the birth path. +import { describe, expect, it } from "vitest"; +import { + TaskRunExecutionSnapshotStore, + type SnapshotStoreMode, + type SnapshotStoreModeResolver, +} from "./taskRunExecutionSnapshotStore.js"; +import type { RedisSnapshotStore } from "./redisSnapshotStore.js"; +import type { RunStore } from "./types.js"; + +const ORG = "org_a"; + +function harness(opts: { + global: SnapshotStoreMode; + forOrg: SnapshotStoreMode; + /** When set, append RESOLVES with this outcome instead of rejecting (the returned-outcome path). */ + appendResult?: { outcome: string; actualCur?: string; seq?: number }; +}) { + const delegateCalls: string[] = []; + const repairs: string[] = []; + + const redis = new Proxy({} as RedisSnapshotStore, { + get: (_t, prop) => { + if (prop === "append") { + return () => + opts.appendResult + ? Promise.resolve(opts.appendResult) + : Promise.reject(new Error("redis append boom")); + } + // No waitpoints in this transition, so #resolveCycle never probes. + return () => Promise.resolve(null); + }, + }); + + const delegate = new Proxy({} as Record, { + get: + (_t, prop) => + (...__: unknown[]) => { + delegateCalls.push(String(prop)); + return Promise.resolve({ id: "run_1" }); + }, + }) as unknown as RunStore; + + const modeResolver: SnapshotStoreModeResolver = { + resolve: (organizationId?: string) => + organizationId === undefined ? opts.global : opts.forOrg, + }; + + const decorated = new TaskRunExecutionSnapshotStore(delegate, { + store: redis, + mode: opts.global, + modeResolver, + onAppendFailure: async ({ runId }) => { + repairs.push(runId); + }, + }); + return { decorated, delegateCalls, repairs }; +} + +function completeParams() { + return [ + "run_1", + { + completedAt: new Date(), + outputType: "application/json", + usageDurationMs: 1, + costInCents: 0, + snapshot: { + id: "snap_1", + executionStatus: "FINISHED", + description: "Run completed", + runStatus: "COMPLETED_SUCCESSFULLY", + attemptNumber: 1, + environmentId: "env_1", + environmentType: "PRODUCTION", + projectId: "proj_1", + organizationId: ORG, + }, + }, + { select: { id: true } }, + ] as never[]; +} + +describe("transition append fatality is decided by the organisation dial", () => { + it("throws and enqueues NO repair when the org resolves to redis-only", async () => { + const { decorated, repairs } = harness({ global: "dual-write", forOrg: "redis-only" }); + + await expect( + (decorated.completeAttemptSuccess as (...a: never[]) => Promise)(...completeParams()) + ).rejects.toThrow(/redis append boom/); + expect(repairs).toEqual([]); + }); + + it("does NOT throw and enqueues a repair below redis-only", async () => { + const { decorated, repairs } = harness({ global: "dual-write", forOrg: "dual-write" }); + + await expect( + (decorated.completeAttemptSuccess as (...a: never[]) => Promise)(...completeParams()) + ).resolves.toBeDefined(); + expect(repairs).toEqual(["run_1"]); + }); +}); + +// The append that RESOLVES with a non-persisting outcome (forked / skippedNoKeyspace) is the gap the +// thrown-error path above always covered: at redis-only the repair reads an empty Postgres, so the +// outcome must be fatal here too, not enqueued for a doomed repair or silently dropped. +describe("a non-persisting append outcome is fatal at redis-only", () => { + it("throws and enqueues NO repair when a transition forks at redis-only", async () => { + const { decorated, repairs } = harness({ + global: "dual-write", + forOrg: "redis-only", + appendResult: { outcome: "forked", actualCur: "snap_other" }, + }); + + await expect( + (decorated.completeAttemptSuccess as (...a: never[]) => Promise)(...completeParams()) + ).rejects.toThrow(/unrecoverable at redis-only/); + expect(repairs).toEqual([]); + }); + + it("throws when a transition is skippedNoKeyspace at redis-only", async () => { + const { decorated, repairs } = harness({ + global: "dual-write", + forOrg: "redis-only", + appendResult: { outcome: "skippedNoKeyspace" }, + }); + + await expect( + (decorated.completeAttemptSuccess as (...a: never[]) => Promise)(...completeParams()) + ).rejects.toThrow(/unrecoverable at redis-only/); + expect(repairs).toEqual([]); + }); + + it("below redis-only, a fork still enqueues a repair and a skip is a no-op", async () => { + const forked = harness({ + global: "dual-write", + forOrg: "dual-write", + appendResult: { outcome: "forked", actualCur: "snap_other" }, + }); + await expect( + (forked.decorated.completeAttemptSuccess as (...a: never[]) => Promise)( + ...completeParams() + ) + ).resolves.toBeDefined(); + expect(forked.repairs).toEqual(["run_1"]); + + const skipped = harness({ + global: "dual-write", + forOrg: "dual-write", + appendResult: { outcome: "skippedNoKeyspace" }, + }); + await expect( + (skipped.decorated.completeAttemptSuccess as (...a: never[]) => Promise)( + ...completeParams() + ) + ).resolves.toBeDefined(); + expect(skipped.repairs).toEqual([]); + }); +}); diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts index e6468c4f09f..cc97db0a3a5 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.transitions.test.ts @@ -365,47 +365,50 @@ describe("transition write ordering", () => { } }); - containerTest( - "reports a forked append without enqueuing a repair", - async ({ prisma, redisOptions }) => { - const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); - try { - const env = await seedSnapshotEnvironment(prisma); - const { workerId, taskId } = await seedSnapshotWorker(prisma, env); - const runId = generateInternalId(); - await seedBirth(decorated, redis, runId, env); + containerTest("asks for a repair when an append forks", async ({ prisma, redisOptions }) => { + const { decorated, redis, repairs, writes } = harness(prisma as never, redisOptions as never); + try { + const env = await seedSnapshotEnvironment(prisma); + const { workerId, taskId } = await seedSnapshotWorker(prisma, env); + const runId = generateInternalId(); + await seedBirth(decorated, redis, runId, env); - // A stale previousSnapshotId: another writer advanced the head. A repair cannot help, so the - // outcome is counted and dropped. - await decorated.lockRunToWorker(runId, { - lockedAt: new Date(), - lockedById: taskId, - lockedToVersionId: workerId, - lockedQueueId: undefined, - startedAt: new Date(), - baseCostInCents: 0, - machinePreset: "small-1x", - taskVersion: "1.0.0", - snapshot: { - id: generateInternalId(), - previousSnapshotId: generateInternalId(), - attemptNumber: 1, - environmentId: env.id, - environmentType: env.type, - projectId: env.projectId, - organizationId: env.organizationId, - completedWaitpointIds: [], - completedWaitpointOrder: [], - }, - }); + // A stale previousSnapshotId: the head is not what this write expected. Every later + // compare-and-set append would fork too, so the run needs the head re-derived from Postgres. + const forkedSnapshotId = generateInternalId(); + await decorated.lockRunToWorker(runId, { + lockedAt: new Date(), + lockedById: taskId, + lockedToVersionId: workerId, + lockedQueueId: undefined, + startedAt: new Date(), + baseCostInCents: 0, + machinePreset: "small-1x", + taskVersion: "1.0.0", + snapshot: { + id: forkedSnapshotId, + previousSnapshotId: generateInternalId(), + attemptNumber: 1, + environmentId: env.id, + environmentType: env.type, + projectId: env.projectId, + organizationId: env.organizationId, + completedWaitpointIds: [], + completedWaitpointOrder: [], + }, + }); - expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); - expect(repairs).toEqual([]); - } finally { - await redis.quit(); - } + expect(writes).toContainEqual({ site: "lockRunToWorker", outcome: "forked" }); + // Exactly one repair, carrying the entry that could not land. One per run, not one per + // transition: the enqueue is deduplicated under the stall watchdog's job id. + expect(repairs).toHaveLength(1); + expect(repairs[0]!.runId).toBe(runId); + expect(repairs[0]!.snapshotId).toBe(forkedSnapshotId); + expect(repairs[0]!.executionStatus).toBe("PENDING_EXECUTING"); + } finally { + await redis.quit(); } - ); + }); containerTest( "appends for the standalone createExecutionSnapshot", diff --git a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts index c6bc145a91d..87cab08f9c4 100644 --- a/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts +++ b/internal-packages/run-store/src/taskRunExecutionSnapshotStore.ts @@ -28,6 +28,7 @@ import { entryFromExpire, entryFromLock, entryFromReschedule, + entryFromSnapshotRow, isTerminalEntry, } from "./snapshotEntry.js"; import { isInjectedFault, type SnapshotFaultInjector } from "./snapshotFaultInjection.js"; @@ -50,6 +51,24 @@ import type { Prisma, PrismaClientOrTransaction, TaskRun } from "@trigger.dev/da /** One initial attempt plus three retries, per the write protocol. */ const APPEND_ATTEMPTS = 4; +// A `forked` or `skippedNoKeyspace` append did NOT persist the transition. Below redis-only that is +// survivable (Postgres holds the head: repair a fork, ignore a legitimate non-resident skip). At +// redis-only Postgres holds nothing, so #recordOutcome throws this instead, mirroring the thrown-error +// fatality the append loops already apply. The loops re-throw it at once rather than retrying, because +// a retry can neither create the keyspace nor unwind a fork. +class RedisOnlyAppendUnrecoverableError extends Error { + constructor(outcome: string) { + super(`snapshot store: append outcome "${outcome}" is unrecoverable at redis-only`); + this.name = "RedisOnlyAppendUnrecoverableError"; + } +} + +function isRedisOnlyAppendUnrecoverable( + error: unknown +): error is RedisOnlyAppendUnrecoverableError { + return error instanceof RedisOnlyAppendUnrecoverableError; +} + /** * Matches the engine's own chunked waitpoint fetch. A batch can complete a thousand waitpoints at * once, and an unbounded `in:` makes each distinct list length its own prepared statement. @@ -68,6 +87,86 @@ const WAITPOINT_CHUNK_SIZE = 100; */ export type SnapshotStoreMode = "off" | "dual-write" | "redis-read" | "redis-only"; +/** + * Resolves the dial for one write. `resolve` MUST be synchronous and MUST NOT query: seven methods + * here take a caller-supplied `tx`, so this class cannot see a caller's transaction boundary, and a + * read issued there could land inside someone else's open interactive transaction. + */ +export type SnapshotStoreModeResolver = { + resolve(organizationId?: string): SnapshotStoreMode; + /** + * Optional, and awaited at BIRTH sites only. Resolves once the organisation's own dial value is + * known to `resolve`, or once the attempt has given up. + * + * Why a birth is different from every other call. The per-organisation dial is served from a + * short-lived cache, and on a miss the resolver answers with the deployment-wide position. For a + * read or a transition that is the right trade. For a birth it is not: residency is decided at + * birth and is permanent, so a run born during a cache miss is excluded from the mirror for its + * entire life and no later pass can adopt it. + * + * That was not theoretical. Three runs born back to back were all resident; after a 14 minute idle + * gap the next one was not, because the cache entry had expired. A miss is any gap longer than the + * cache lifetime, so on bursty traffic the first run of every burst was silently excluded, and a + * low-traffic canary organisation would have lost most of its runs. + * + * Implementations MUST bound this. A birth is on the trigger path and a caller may already hold an + * open transaction, so a slow flag read must give up and leave `resolve` answering as before + * rather than hold that transaction open. Failing is always allowed: the fallback is the previous + * behaviour, never an error. + */ + warm?(organizationId: string): Promise; + /** + * Optional one-way global latch: has the deployment-wide dial EVER been non-off. False means the + * global dial has never moved, so nothing was born resident by the global position; combined with a + * definite per-org negative it is what lets a transition skip the keyspace probe. It is what makes + * `off` genuinely inert rather than merely quiet: measured at 2 per cent with a healthy endpoint + * and four times the run duration with a slow one, for every run, with no decay. + * + * MUST be synchronous and MUST NOT query. Conservative when cold: absent or true means "do not + * skip", so any uncertainty keeps probing. + */ + globalModeEverEnabled?(): boolean; + /** + * Optional per-organisation definite negative: is this organisation DEFINITELY never-enabled, i.e. + * a source that is loaded AND has never seen this org enabled. True is the other half of the sound + * skip: with the global dial unmoved and this org definitely never-enabled, no birth of its runs + * ever mirrored, so no keyspace exists and a transition would be refused anyway. + * + * MUST be synchronous and MUST NOT query. Absent, false, or unknown (cold source) all mean "do not + * skip", so any uncertainty keeps probing. + */ + orgDefinitelyNeverEnabled?(organizationId: string): boolean; + /** + * Optional. The effective READ position for one run, org-scoped so a single org can be soaked at + * `redis-read` while everyone else stays on Postgres. Same contract as `resolve`: synchronous, MUST + * NOT query. Absent, or unresolved, falls back to the global mode, which is safe during soak. + */ + readModeFor?(runId: string, environmentId?: string): SnapshotStoreMode | undefined; + /** + * Optional, ASYNC authoritative counterpart to `readModeFor`. Called only when `readModeFor` is + * unresolved AND some org is `redis-only`, to decide whether a Postgres fallback would strand the + * run. Bounded and MAY throw; the caller fails closed (no fallback) on a throw or when absent. + */ + readModeForAuthoritative?( + runId: string, + environmentId?: string + ): Promise; + /** + * Optional, cheap. Is ANY organisation currently at `redis-read` or `redis-only`. When false and + * the global dial is not itself at a read position, a read short-circuits to Postgres without + * resolving the run's org, keeping the dual-write soak phase at zero new read cost. Same contract: + * synchronous, MUST NOT query. + */ + anyOrgReadEnabled?(): boolean; + /** + * Optional, cheap. Is ANY organisation currently at `redis-only`. Governs the fallback gate when a + * run's org cannot be resolved: with some org at `redis-only`, a Redis error on an unresolved run + * must throw rather than serve an empty Postgres, because that org holds its snapshots nowhere + * else. Same contract: synchronous, MUST NOT query. + */ + anyOrgRedisOnly?(): boolean; +}; + /** * Enqueues the existing `repairSnapshot` job for a run whose append was lost. The decorator lives in * run-store and cannot reach the engine's worker, so the binding is injected. That binding must @@ -80,9 +179,43 @@ export type SnapshotRepairEnqueuer = (args: { executionStatus: string; }) => Promise; +/** + * The hard stop, orthogonal to the dial because it answers a different question: the dial says how + * far the rollout has got, this says whether to write at all right now. Re-read on every write so it + * can be thrown without a deploy. + * + * Throwing it freezes every resident run's Redis head while Postgres advances, so it is a + * resync-before-reads control, not a rollback. `off` is the lossless way down. + */ +export type SnapshotStoreHaltCheck = () => boolean; + +/** + * What one repair attempt did. Bounded so it can be a metric tag, and every value is an outcome the + * caller only logs: no value here means the run needs anything further done to it. + */ +export type SnapshotRepairOutcome = + | "halted" + | "noSnapshot" + | "notResident" + | "alreadyCurrent" + | "redisAhead" + | "reappended" + | "duplicate" + | "forked"; + +/** The two seams the snapshot repair job needs, present only when the mirror is wired up. */ +export type SnapshotMirrorRepair = { + authoritativeStore(): RunStore; + repairRedisHead(runId: string, snapshotId: string): Promise; +}; + +export function asSnapshotMirrorRepair(store: RunStore): SnapshotMirrorRepair | undefined { + return store instanceof TaskRunExecutionSnapshotStore ? store : undefined; +} + export type DecoratorMetrics = { recordWrite(site: string, outcome: string): void; - recordAppendFailed(site: string): void; + recordAppendFailed(site: string, organizationId?: string): void; recordRead(method: string, source: "redis" | "postgres"): void; }; @@ -90,8 +223,12 @@ export type TaskRunExecutionSnapshotStoreOptions = { store: RedisSnapshotStore; /** Defaults to `off`, which is a pure pass-through that never touches Redis. */ mode?: SnapshotStoreMode; + /** Takes precedence over `mode`, and is re-read on every write so the dial can move at runtime. */ + modeResolver?: SnapshotStoreModeResolver; /** Percentage of runs whose reads come from Redis at `redis-read` and `redis-only`. Defaults to 0. */ readPercent?: number; + /** Defaults to never halted. */ + halted?: SnapshotStoreHaltCheck; onAppendFailure?: SnapshotRepairEnqueuer; faults?: SnapshotFaultInjector; metrics?: DecoratorMetrics; @@ -117,9 +254,11 @@ export type StagedAppend = { }; export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { - readonly mode: SnapshotStoreMode; + readonly #staticMode: SnapshotStoreMode; + protected readonly modeResolver?: SnapshotStoreModeResolver; protected readonly redis: RedisSnapshotStore; protected readonly readPercent: number; + protected readonly haltCheck?: SnapshotStoreHaltCheck; protected readonly onAppendFailure?: SnapshotRepairEnqueuer; protected readonly faults?: SnapshotFaultInjector; protected readonly metrics?: DecoratorMetrics; @@ -129,8 +268,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { constructor(delegate: RunStore, options: TaskRunExecutionSnapshotStoreOptions) { super(delegate); this.redis = options.store; - this.mode = options.mode ?? "off"; + this.#staticMode = options.mode ?? "off"; + this.modeResolver = options.modeResolver; this.readPercent = options.readPercent ?? 0; + this.haltCheck = options.halted; this.onAppendFailure = options.onAppendFailure; this.faults = options.faults; this.metrics = options.metrics; @@ -138,9 +279,119 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { this.staging = options.staging; } - /** True in every position that appends to Redis. */ - protected get writesRedis(): boolean { - return this.mode !== "off"; + get mode(): SnapshotStoreMode { + return this.modeResolver?.resolve() ?? this.#staticMode; + } + + /** The resolved position for one organisation. Falls back to the global answer when unknown. */ + protected modeFor(organizationId?: string): SnapshotStoreMode { + return this.modeResolver?.resolve(organizationId) ?? this.#staticMode; + } + + /** Whether the hard stop is thrown. A halt beats every dial position, in both directions. */ + protected halted(): boolean { + try { + return this.haltCheck?.() === true; + } catch { + // An unreadable switch must not decide anything. Halting on an error would freeze every + // resident run's head, which is the outcome this whole area exists to avoid. + return false; + } + } + + /** + * Whether a BIRTH mirrors to Redis. This is the only decision the per-organisation override gets + * to make, and it fixes the run's store for the rest of its life. + */ + /** + * Waits for the organisation's real dial value before a birth decides residency. Never throws: + * a flag read that fails or times out leaves `resolve` answering exactly as it did before, which + * is the behaviour this replaces. + */ + async #warmOrgMode(organizationId?: string): Promise { + if (!organizationId || !this.modeResolver?.warm) { + return; + } + try { + await this.modeResolver.warm(organizationId); + } catch (error) { + this.logger.warn("snapshot store could not warm the organisation dial before a birth", { + organizationId, + error, + }); + } + } + + protected writesRedisForBirth(organizationId?: string): boolean { + if (this.halted()) return false; + + return this.modeFor(organizationId) !== "off"; + } + + /** + * Whether a TRANSITION mirrors to Redis. Deliberately blind to the organisation. + * + * A transition belongs to a run that is already resident or already absent, and the append script + * refuses a transition into a keyspace that does not exist. So the keyspace IS the per-run + * residency record, and asking the organisation again would only introduce the one thing the + * design forbids: a run changing stores half way through its life. That is not a hypothetical. + * The override is served from a short-lived cache that falls back to the deployment-wide position + * on a miss, so a run could be born into Redis during one window and have its next transitions + * refused in the next, freezing its head while Postgres moved on, with no fault and no log line. + * + * The deployment-wide dial is blind here for the same reason. An operator turning it down to `off` + * mid-incident would otherwise inflict that identical freeze on every resident run at once, which + * makes the remedy indistinguishable from the fault. `off` therefore stops new residency only: + * births stop, resident runs keep mirroring, and the mirror drains as they finish. Stopping + * outright is the halt switch, and it is a resync control rather than a rollback. + */ + protected writesRedisForTransition(organizationId?: string): boolean { + if (this.halted()) { + return false; + } + + // The ONLY case where a transition may be skipped without a keyspace check. Soundness invariant: + // a run is resident only if modeFor(org) != off at its birth, i.e. the org had an override OR the + // global dial was non-off. So skipping is safe only when the global dial was NEVER non-off + // (globalModeEverEnabled() === false) AND this org is DEFINITELY never-enabled + // (orgDefinitelyNeverEnabled(org) === true). With both true, no birth of this org's runs ever + // mirrored, so no keyspace exists and the append script would refuse every one of these anyway. + // Any uncertainty (cold registry or census, absent signal, undefined org) leaves a guard + // unsatisfied and the transition probes: fail-safe toward asking. + // + // Accepted transient at the enabling edge, not made airtight: the census is eventually + // consistent, so a just-enabled org may briefly still read as definitely-never-enabled and a + // transition could skip. It is recoverable (the census reloads within its interval and later + // transitions mirror again) and harmless at dual-write (Postgres is authoritative; an org reaches + // a read position only after soak far longer than census convergence). The enabling save refreshes + // the census in its own process to shrink the window; other pods lag at most the reload interval. + if ( + organizationId !== undefined && + this.modeResolver?.globalModeEverEnabled?.() === false && + this.modeResolver?.orgDefinitelyNeverEnabled?.(organizationId) === true + ) { + return false; + } + + return true; + } + + /** Test seams for the two predicates. Not for production callers. */ + writesRedisForBirthTest(organizationId?: string): boolean { + return this.writesRedisForBirth(organizationId); + } + + writesRedisForTransitionTest(organizationId?: string): boolean { + return this.writesRedisForTransition(organizationId); + } + + haltedTest(): boolean { + return this.halted(); + } + + /** Test seam for the resolved position. Not for production callers. */ + modeForTest(organizationId?: string): SnapshotStoreMode { + return this.modeFor(organizationId); } /** @@ -159,12 +410,9 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { runId: string | undefined, fn: (store: RunStore, tx: PrismaClientOrTransaction) => Promise ): Promise { - if (!this.writesRedis) { - // At `off` the callback must receive the delegate's own store, untouched, so a transaction - // behaves exactly as it does without the decorator in the chain. - return this.delegate.runInTransaction(runId, fn); - } - + // Always stage. This method holds only a runId, so it cannot know whether the writes inside + // belong to a resident run. A birth inside the callback still resolves its own organisation's + // dial as it is staged; a transition stages unconditionally, as it does outside a transaction. const staged: StagedAppend[] = []; const result = await this.delegate.runInTransaction(runId, (store, tx) => @@ -195,10 +443,6 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ): Promise { const store = await this.delegate.forWaitpointCompletion(waitpointId, context); - if (!this.writesRedis) { - return store; - } - // Carry the staging buffer through. Without it, a handle taken inside a transaction appends // immediately, which is the exact ordering the facade exists to prevent. return this.#wrap(store, this.staging); @@ -212,8 +456,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { #wrap(store: RunStore, staging?: StagedAppend[]): TaskRunExecutionSnapshotStore { return new TaskRunExecutionSnapshotStore(store, { store: this.redis, - mode: this.mode, + mode: this.#staticMode, readPercent: this.readPercent, + ...(this.haltCheck && { halted: this.haltCheck }), + ...(this.modeResolver && { modeResolver: this.modeResolver }), logger: this.logger, ...(this.onAppendFailure && { onAppendFailure: this.onAppendFailure }), ...(this.faults && { faults: this.faults }), @@ -230,7 +476,21 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { params: CreateRunInput, tx?: PrismaClientOrTransaction ): Promise { - if (!this.writesRedis) { + // Before the decision, not after: residency is permanent, so this is the one call where the + // organisation's real value is worth waiting for. + await this.#warmOrgMode(params.snapshot?.organizationId); + + if (!this.writesRedisForBirth(params.snapshot?.organizationId)) { + // Deliberately records NOTHING about residency here, though it is tempting: this run is + // almost certainly non-resident, and seeding that would save its first transition a probe. + // + // It is not safe. A birth path can be re-entered (createCancelledRun has an explicit + // "row already exists" path), so a birth that DID mirror on its first attempt can reach here + // on a retry once the short-lived override cache has moved to off. Seeding a negative from a + // local decision would then suppress every later transition of a run that is resident, + // freezing its head while Postgres moved on: the exact failure the residency model exists to + // prevent. Only the append script's own reply is authoritative about a keyspace, so only that + // reply may create a negative. return this.delegate.createRun(params, tx); } @@ -246,7 +506,9 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { params: CreateCancelledRunInput, tx?: PrismaClientOrTransaction ): Promise { - if (!this.writesRedis) { + await this.#warmOrgMode(params.snapshot?.organizationId); + + if (!this.writesRedisForBirth(params.snapshot?.organizationId)) { return this.delegate.createCancelledRun(params, tx); } @@ -275,7 +537,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { args: { select: S }, tx?: PrismaClientOrTransaction ): Promise> { - if (!this.writesRedis) { + if (!this.writesRedisForTransition(data.snapshot?.organizationId)) { return this.delegate.completeAttemptSuccess(runId, data, args, tx); } @@ -300,7 +562,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { args: { select: S }, tx?: PrismaClientOrTransaction ): Promise> { - if (!this.writesRedis) { + if (!this.writesRedisForTransition(data.snapshot?.organizationId)) { return this.delegate.expireRun(runId, data as never, args, tx); } @@ -327,7 +589,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { }, tx?: PrismaClientOrTransaction ): Promise<{ count: number }> { - if (!this.writesRedis) { + if (!this.writesRedisForTransition(data.snapshot?.organizationId)) { return this.delegate.expireParkedRun(runId, data as never, tx); } @@ -353,7 +615,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ): Promise { // The delegate writes a snapshot only when one is supplied, so an absent snapshot is a plain run // update with nothing for Redis to mirror. - if (!this.writesRedis || !data.snapshot) { + if (!data.snapshot || !this.writesRedisForTransition(data.snapshot?.organizationId)) { return this.delegate.rescheduleRun(runId, data, tx); } @@ -374,7 +636,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { data: LockRunData, tx?: PrismaClientOrTransaction ): Promise>> { - if (!this.writesRedis) { + if (!this.writesRedisForTransition(data.snapshot?.organizationId)) { return this.delegate.lockRunToWorker(runId, data, tx); } @@ -404,7 +666,7 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { input: CreateExecutionSnapshotInput, tx?: PrismaClientOrTransaction ): Promise> { - if (!this.writesRedis) { + if (!this.writesRedisForTransition(input.organizationId)) { return this.delegate.createExecutionSnapshot(input, tx); } @@ -462,19 +724,24 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { kind: "birth", isTerminal: isTerminalEntry(entry), }); - this.#recordOutcome(site, entry, result); + await this.#recordOutcome(site, entry, result); // Modelled AFTER the successful append: the crash this boundary represents is a process that // died between the two stores, not an append that failed. this.faults?.("afterRedisBirthBeforePg", { runId: entry.runId, snapshotId: entry.id }); return; } catch (error) { + // A non-persisting outcome at redis-only is fatal; never retry it (a retry cannot create the + // keyspace), so run creation fails loudly rather than being born unrecorded. + if (isRedisOnlyAppendUnrecoverable(error)) { + throw error; + } if (isInjectedFault(error)) { throw error; } if (attempt === APPEND_ATTEMPTS - 1) { - this.metrics?.recordAppendFailed(site); + this.metrics?.recordAppendFailed(site, entry.organizationId); this.logger.error("snapshot birth append failed after retries", { runId: entry.runId, snapshotId: entry.id, @@ -483,7 +750,10 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { error, }); - if (this.mode === "redis-only") { + // The same organisation dial that decided to append decides whether a lost birth is + // fatal. Using the global position here would fail run creation for an organisation whose + // own position still has Postgres authoritative. + if (this.modeFor(entry.organizationId) === "redis-only") { throw error; } return; @@ -533,24 +803,37 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ...(cycle && { cycle }), }); - this.#recordOutcome(site, entry, result); + await this.#recordOutcome(site, entry, result); return; } catch (error) { + // A non-persisting outcome at redis-only is fatal: never retry it (the keyspace cannot appear + // and a fork cannot unwind), surface it at once. + if (isRedisOnlyAppendUnrecoverable(error)) { + throw error; + } // An injected fault models a dead process, not a retryable append failure. if (isInjectedFault(error)) { - this.metrics?.recordAppendFailed(site); + this.metrics?.recordAppendFailed(site, entry.organizationId); + // At redis-only Postgres holds nothing, so a lost transition is unrecoverable and the + // repair cannot help: surface it instead, as the birth path does. + if (this.modeFor(entry.organizationId) === "redis-only") { + throw error; + } await this.#enqueueRepair(entry); return; } if (attempt === APPEND_ATTEMPTS - 1) { - this.metrics?.recordAppendFailed(site); + this.metrics?.recordAppendFailed(site, entry.organizationId); this.logger.error("snapshot append failed after retries", { runId: entry.runId, snapshotId: entry.id, site, error, }); + if (this.modeFor(entry.organizationId) === "redis-only") { + throw error; + } await this.#enqueueRepair(entry); return; } @@ -560,6 +843,102 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } + /** + * The undecorated store. A repair MUST read through this and not through `this`: once reads are + * served from Redis, reading through the decorator hands the repair the very stale head it exists + * to replace, so it sees the id it was asked about missing and concludes there is nothing to do. + */ + authoritativeStore(): RunStore { + return this.delegate; + } + + /** + * Re-appends the Postgres head of a run whose mirror append was lost. + * + * Additive only, and neither of the append script's guards is reimplemented here: an entry that + * already landed comes back `duplicate`, and a run with no keyspace comes back + * `skippedNoKeyspace`, which is how a run that was never resident stays non-resident. + * + * No compare-and-set: asserting cur would refuse exactly when the mirror is furthest behind and a + * stale head is doing the most damage. The one ordering rule kept is that the repair will not + * append behind a mirror head already newer than Postgres. + */ + async repairRedisHead(runId: string, snapshotId: string): Promise { + if (!this.writesRedisForTransition()) { + // The hard stop, not the dial: the dial governs births and never refuses a repair. + return "halted"; + } + + const row = await this.delegate.findLatestExecutionSnapshot(runId); + + if (!row) { + return "noSnapshot"; + } + + // The target is the head Postgres holds NOW, not the snapshot the job names. The repair is + // delayed, so the run has usually moved on by the time it runs, and the entry that was lost is + // unreachable by then; the mirror head is still wrong, and a wrong head is what a Redis-served + // read returns. + if (row.id !== snapshotId) { + this.logger.log("snapshot repair target advanced", { + runId, + enqueuedFor: snapshotId, + head: row.id, + }); + } + + const head = await this.redis.getLatest(runId); + + if (head?.id === row.id) { + // The head converged on its own, but a repair only runs because an append was LOST, so the + // entries behind that head can still be missing. This is the case observed live: four entries + // in Redis against eight in Postgres, with a matching head. Returning here without marking is + // what let a short window be served as though it were whole. + await this.redis.markGapsIfResident(runId); + return "alreadyCurrent"; + } + + if (head && headIsNewerThan(head, row.createdAt)) { + // Redis ahead of the Postgres head is divergence too, whichever way round it is. + await this.redis.markGapsIfResident(runId); + return "redisAhead"; + } + + const entry = entryFromSnapshotRow(row); + const refs = lockCycleRefs( + row.completedWaitpoints.map((waitpoint) => waitpoint.id), + row.completedWaitpointOrder + ); + const cycle = await this.#resolveCycle(runId, refs); + + const result = await this.redis.append({ + entry, + kind: "transition", + isTerminal: isTerminalEntry(entry), + // A repair runs BECAUSE an append was lost. Whatever it manages to put back, the entries in + // between are gone for good, and a window read cannot detect a hole. So the keyspace is marked + // and its windows fall back to Postgres, which still holds the whole log. Backfilling the lost + // entries instead would be worse: a late append takes a fresh seq, and the window scripts walk + // the index in seq order as though it were time order, so an old entry with a high seq + // truncates the window harder than the hole does. + markGaps: true, + ...(cycle && { cycle }), + }); + + this.metrics?.recordWrite("repairRedisHead", result.outcome); + + switch (result.outcome) { + case "written": + return "reappended"; + case "duplicate": + return "duplicate"; + case "skippedNoKeyspace": + return "notResident"; + case "forked": + return "forked"; + } + } + /** * Decides whether this append mints a new wait cycle or points at the one already there. * @@ -620,29 +999,80 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } /** - * None of the four append outcomes is a failure, and none of them enqueues a repair. + * No append outcome is a thrown failure. Exactly one of them enqueues a repair. * - * `skippedNoKeyspace` is every pre-cutover run's transitions. `forked` means another writer - * advanced the head, which a repair cannot help. `duplicate` is a retry that already landed. - * `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + * `skippedNoKeyspace` is a run that is not resident: every pre-cutover run's transitions, and + * every run whose organisation was at `off` when it was born. `duplicate` is a retry that already + * landed. `cycleMismatch` means the store refused an untrustworthy waitpoint pointer on purpose. + * None of those three is a fault, and none enqueues anything. + * + * `forked` is none of those. It was read as expected contention, from a model where any writer + * could append to any run. With a run's store fixed at birth the writer set per run is stable, so + * a fork is either a lost append or a genuinely concurrent writer, and by the time it is seen the + * head already disagrees with Postgres. + * + * A fork therefore enqueues a repair as well as paging. It did not, once, and that was a defect: + * the two compare-and-set sites assert the head, so a head left wrong makes every later append + * from those sites fork too, and the mirror stays frozen for the rest of the run's life. Nothing + * else clears it. The sweep will not: it skips live runs by design. + * + * Two earlier attempts at Postgres-driven head repair in this area were withdrawn as unsafe, and + * this is not a third. Both asserted `cur` while re-appending, so they could fork on the very + * condition they were sent to fix. This repair asserts nothing: it re-derives the head from + * Postgres, refuses when the mirror head is already current or demonstrably newer, and marks the + * keyspace so window reads fall back rather than serve the entries the fork lost. See the + * SnapshotStoreAppendForked rule, which now reads as "divergence happened" rather than "divergence + * persists". */ - #recordOutcome( + async #recordOutcome( site: string, entry: SnapshotEntryInput, result: Awaited> - ): void { + ): Promise { this.metrics?.recordWrite(site, result.outcome); + // At redis-only Postgres holds no snapshot for this run, so an outcome that did not persist the + // transition is unrecoverable: a fork cannot be repaired from an empty Postgres, and a + // skippedNoKeyspace has nowhere else to land. Surface a retryable failure rather than enqueue a + // doomed repair or silently drop it, matching the thrown-error fatality in the append loops. + if ( + (result.outcome === "forked" || result.outcome === "skippedNoKeyspace") && + this.modeFor(entry.organizationId) === "redis-only" + ) { + this.logger.error("snapshot append did not persist at redis-only", { + runId: entry.runId, + snapshotId: entry.id, + site, + outcome: result.outcome, + }); + throw new RedisOnlyAppendUnrecoverableError(result.outcome); + } + if (result.outcome === "forked") { - this.logger.warn("snapshot append forked", { + this.logger.error("snapshot append forked", { runId: entry.runId, snapshotId: entry.id, site, actualCur: result.actualCur, }); + + // A fork is not a race to shrug at: the two compare-and-set sites assert the head, so once it + // is wrong every later append from them forks too and the mirror is frozen for the rest of the + // run. The repair re-derives the head from Postgres without asserting `cur`, which is the one + // operation that clears this, so ask for it rather than only reporting. + await this.#enqueueRepair(entry); } } + /** Test seam for the outcome handler. Not for production callers. */ + async recordOutcomeForTest( + site: string, + entry: SnapshotEntryInput, + result: Awaited> + ): Promise { + await this.#recordOutcome(site, entry, result); + } + // --------------------------------------------------------------------------------------------- // Reads. // @@ -656,13 +1086,97 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { * Whether this run's reads come from Redis. Hashed on the run id so a run does not change store * between two reads of the same poll, which would let a caller see the log go backwards. */ - protected readsFromRedis(runId: string): boolean { - if (this.mode !== "redis-read" && this.mode !== "redis-only") return false; + /** + * Whether a failed Redis read may be answered from Postgres. + * + * The read paths fell back on a MISS and on a dangling cycle, but not on an ERROR, so a brownout + * at `redis-read` turned an engine read into a throw once the command timed out. Postgres still + * holds every row below `redis-only`, so falling back is strictly better than failing. + * + * At `redis-only` it is not an option: nothing else holds the rows, so the error is the answer and + * hiding it would serve an empty history as though it were real. Org-scoped, so a single org soaked + * at `redis-only` throws while everyone else still falls back. + */ + async #fallbackAllowed(runId?: string, environmentId?: string): Promise { + // The global dial preserves today's behaviour exactly: at redis-only Postgres holds nothing. + if (this.mode === "redis-only") return false; + + // The run's own org from the sync cache, checked BEFORE the census: a run whose org resolves to + // redis-only must throw even when anyOrgRedisOnly (a separate, independently-lagging source) has + // not yet caught the enable, or the writing-process window strands it. A concrete non-`redis-only` + // mode falls back normally, even when some OTHER org is `redis-only`. + if (runId !== undefined) { + const resolved = this.modeResolver?.readModeFor?.(runId, environmentId); + if (resolved === "redis-only") return false; + if (resolved !== undefined) return true; + } + + // Org unresolved (cold run→org cache, or no runId). If no org is `redis-only`, this run cannot + // be either, so Postgres is a valid answer. + if (this.modeResolver?.anyOrgRedisOnly?.() !== true) return true; + + // Some org IS redis-only and the sync cache cannot tell if it is this one. Resolve the run's org + // authoritatively rather than strand a redis-only run or over-throw a pre-cutover one. An absent + // hook, an undefined answer, or a bounded read that fails all fail closed: a retryable throw beats + // serving an empty Postgres when some org genuinely has nowhere else. + // + // A missing runId cannot be attributed to a redis-only org, and every engine read of a resident + // run threads one; global redis-only is already handled above. So fall back rather than over-throw + // the run-id-less fan-out that only exists below redis-only. + if (runId === undefined) return true; + try { + const authoritative = await this.modeResolver?.readModeForAuthoritative?.( + runId, + environmentId + ); + if (authoritative === undefined) return false; + return authoritative !== "redis-only"; + } catch { + return false; + } + } + + // Thrown when a redis-only run cannot fall back to Postgres (a miss, a dangling cycle, or a routed + // read Redis cannot serve). Retryable by design: the head may still be catching up, and serving an + // empty Postgres would strand the run. + #redisOnlyMissError(): Error { + return new Error("snapshot store: run is redis-only, so Postgres cannot serve this read"); + } + + #reportReadUnavailable(method: string, runId: string, error: unknown): void { + this.logger.warn("snapshot read fell back to Postgres after a store error", { + method, + runId, + error, + }); + this.metrics?.recordRead(method, "postgres"); + } + + protected readsFromRedis(runId: string, environmentId?: string): boolean { + // Short-circuit before any org resolution: while no org is read-enabled and the global dial is + // not itself at a read position, nothing reads from Redis, so skip resolving the run's org + // entirely. This is what keeps the dual-write soak phase at zero new read cost. + if ( + this.modeResolver?.anyOrgReadEnabled?.() !== true && + this.mode !== "redis-read" && + this.mode !== "redis-only" + ) { + return false; + } + + // The org-scoped read position, falling back to the global answer when unresolved. + const effective = this.modeResolver?.readModeFor?.(runId, environmentId) ?? this.mode; + + if (effective !== "redis-read" && effective !== "redis-only") return false; // At `redis-only` the cohort dial has no meaning. Postgres holds no snapshot rows at that // position, so a run routed away from Redis reads nothing at all. Ignoring the percentage here // makes that misconfiguration unreachable rather than merely documented. - if (this.mode === "redis-only") return true; + if (effective === "redis-only") return true; + + // Halted heads are frozen, and below `redis-only` Postgres still holds the whole log, so serving + // reads from it is strictly better than serving a head that stopped moving. + if (this.halted()) return false; if (this.readPercent >= 100) return true; if (this.readPercent <= 0) return false; @@ -681,13 +1195,23 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ): Promise | null> { - if (!this.readsFromRedis(runId)) { + if (!this.readsFromRedis(runId, environmentId)) { + if (!(await this.#fallbackAllowed(runId, environmentId))) throw this.#redisOnlyMissError(); return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } - const read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + let read; + try { + read = await this.redis.getLatest(runId, { ...(environmentId && { environmentId }) }); + } catch (error) { + if (!(await this.#fallbackAllowed(runId, environmentId))) throw error; + this.#reportReadUnavailable("findLatestExecutionSnapshot", runId, error); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } if (!read) { - // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error. + // A miss is the coexistence path: a pre-cutover run, or expired history. It is not an error — + // unless the run is redis-only, where Postgres holds nothing, so a miss must throw not delegate. + if (!(await this.#fallbackAllowed(runId, environmentId))) throw this.#redisOnlyMissError(); this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } @@ -695,13 +1219,27 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { if (read.danglingCycle) { // The entry says it has waitpoints and the cycle key holding them is gone. Serving it would // hand back an empty set that looks authoritative, and the run would resume with no waits. - // Postgres still has the join rows. + // Postgres still has the join rows — unless the run is redis-only, where it does not. + if (!(await this.#fallbackAllowed(runId, environmentId))) throw this.#redisOnlyMissError(); this.metrics?.recordRead("findLatestExecutionSnapshot", "postgres"); return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); } - this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); - return this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); + // Recorded AFTER hydration, not before. Hydration can fall back to Postgres, and recording + // `redis` up front then `postgres` on the way out made one logical read increment both series. + // Hydration is inside the boundary too. It makes a SECOND Redis call when the entry has a wait + // cycle whose ids the read did not carry, and a failure there is the same brownout the catch + // above exists for. Leaving it outside meant a waitpoint-bearing run still threw into the + // engine while a plain one fell back. + try { + const hydrated = await this.#hydrate(read, runId, client, { hydrateWaitpointRows: true }); + this.metrics?.recordRead("findLatestExecutionSnapshot", "redis"); + return hydrated; + } catch (error) { + if (!(await this.#fallbackAllowed(runId, environmentId))) throw error; + this.#reportReadUnavailable("findLatestExecutionSnapshot", runId, error); + return this.delegate.findLatestExecutionSnapshot(runId, client, environmentId); + } } override async findExecutionSnapshot( @@ -709,15 +1247,30 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { client?: ReadClient ): Promise | null> { const shape = matchSinceCursorLookup(args); - if (!shape || !this.readsFromRedis(shape.runId)) { + if (!shape) { + // A query shape Redis cannot serve. It is not a redis-only miss, so it delegates unguarded. + return this.delegate.findExecutionSnapshot(args, client); + } + if (!this.readsFromRedis(shape.runId, shape.environmentId)) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) + throw this.#redisOnlyMissError(); return this.delegate.findExecutionSnapshot(args, client); } - const found = await this.redis.getById(shape.runId, shape.id, { - ...(shape.environmentId && { environmentId: shape.environmentId }), - }); + let found; + try { + found = await this.redis.getById(shape.runId, shape.id, { + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + } catch (error) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) throw error; + this.#reportReadUnavailable("findExecutionSnapshot", shape.runId, error); + return this.delegate.findExecutionSnapshot(args, client); + } if (!found) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) + throw this.#redisOnlyMissError(); this.metrics?.recordRead("findExecutionSnapshot", "postgres"); return this.delegate.findExecutionSnapshot(args, client); } @@ -734,35 +1287,61 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { client?: ReadClient ): Promise[]> { const shape = matchSinceWindow(args); - if (!shape || !this.readsFromRedis(shape.runId)) { + if (!shape) { + // A query shape Redis cannot serve. It is not a redis-only miss, so it delegates unguarded. + return this.delegate.findManyExecutionSnapshots(args, client); + } + if (!this.readsFromRedis(shape.runId, shape.environmentId)) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) + throw this.#redisOnlyMissError(); return this.delegate.findManyExecutionSnapshots(args, client); } - const result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { - limit: shape.take, - ...(shape.environmentId && { environmentId: shape.environmentId }), - }); + let result; + try { + result = await this.redis.getSinceCreatedAt(shape.runId, shape.createdAt, { + limit: shape.take, + ...(shape.environmentId && { environmentId: shape.environmentId }), + }); + } catch (error) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) throw error; + this.#reportReadUnavailable("findManyExecutionSnapshots", shape.runId, error); + return this.delegate.findManyExecutionSnapshots(args, client); + } if (result.kind === "miss") { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) + throw this.#redisOnlyMissError(); this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); return this.delegate.findManyExecutionSnapshots(args, client); } if (result.entries.some((entry) => entry.danglingCycle)) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) + throw this.#redisOnlyMissError(); this.metrics?.recordRead("findManyExecutionSnapshots", "postgres"); return this.delegate.findManyExecutionSnapshots(args, client); } - this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); - // The engine asks for createdAt DESC and reverses app-side; the store returns ascending. const descending = [...result.entries].reverse(); // Rows are hydrated for no entry here: the engine fetches the head's waitpoints itself, from // the ids this call's head row reports. Each row still carries its own order. - const hydrated = await Promise.all( - descending.map((entry) => this.#hydrate(entry, shape.runId, client)) - ); - return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + // + // Inside the boundary for the same reason as the sibling read: hydration can make a second Redis + // call, and one window entry failing it must fall back rather than throw. + try { + const hydrated = await Promise.all( + descending.map((entry) => this.#hydrate(entry, shape.runId, client)) + ); + // Recorded once every entry hydrated, for the same reason as the sibling read. + this.metrics?.recordRead("findManyExecutionSnapshots", "redis"); + return hydrated as unknown as Prisma.TaskRunExecutionSnapshotGetPayload[]; + } catch (error) { + if (!(await this.#fallbackAllowed(shape.runId, shape.environmentId))) throw error; + this.#reportReadUnavailable("findManyExecutionSnapshots", shape.runId, error); + return this.delegate.findManyExecutionSnapshots(args, client); + } } override async findSnapshotCompletedWaitpointIds( @@ -772,11 +1351,20 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { ): Promise { // Without a run id there is no keyspace to look in, so the router's fan-out is the only answer. if (!runId || !this.readsFromRedis(runId)) { + if (!(await this.#fallbackAllowed(runId))) throw this.#redisOnlyMissError(); return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); } - const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + let ids; + try { + ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + } catch (error) { + if (!(await this.#fallbackAllowed(runId))) throw error; + this.#reportReadUnavailable("findSnapshotCompletedWaitpointIds", runId, error); + return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); + } if (!ids.present) { + if (!(await this.#fallbackAllowed(runId))) throw this.#redisOnlyMissError(); this.metrics?.recordRead("findSnapshotCompletedWaitpointIds", "postgres"); return this.delegate.findSnapshotCompletedWaitpointIds(snapshotId, client, runId); } @@ -791,13 +1379,23 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { runId?: string ): Promise<{ present: boolean; ids: string[] }> { if (!runId || !this.readsFromRedis(runId)) { + if (!(await this.#fallbackAllowed(runId))) throw this.#redisOnlyMissError(); return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); } - const ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + let ids; + try { + ids = await this.redis.getSnapshotWaitpointIds(runId, snapshotId); + } catch (error) { + if (!(await this.#fallbackAllowed(runId))) throw error; + this.#reportReadUnavailable("findSnapshotCompletedWaitpointIdsWithPresence", runId, error); + return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); + } if (!ids.present) { // present=false means this reader cannot see the snapshot, so its empty list is not - // authoritative and the engine's read-repair needs the Postgres answer. + // authoritative and the engine's read-repair needs the Postgres answer — except at redis-only, + // where Postgres holds nothing, so the miss must throw rather than serve an empty set. + if (!(await this.#fallbackAllowed(runId))) throw this.#redisOnlyMissError(); this.metrics?.recordRead("findSnapshotCompletedWaitpointIdsWithPresence", "postgres"); return this.delegate.findSnapshotCompletedWaitpointIdsWithPresence(snapshotId, client, runId); } @@ -946,6 +1544,19 @@ export class TaskRunExecutionSnapshotStore extends DelegatingRunStore { } } +/** + * A mirror head with no readable timestamp cannot be shown to be newer, so the repair proceeds. The + * alternative is refusing on an unparseable value, which would strand the run with a stale head. + */ +function headIsNewerThan(head: SnapshotRead, createdAt: Date): boolean { + const raw = head.entry["createdAt"]; + if (typeof raw !== "string") { + return false; + } + const headMs = Date.parse(raw); + return Number.isFinite(headMs) && headMs > createdAt.getTime(); +} + /** Position-sensitive: the same ids in a different order are a different wait cycle. */ function sameOrder(a: string[], b: string[]): boolean { return a.length === b.length && a.every((id, index) => id === b[index]); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c9442bfcb1..8641cd03262 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1299,6 +1299,9 @@ importers: '@trigger.dev/database': specifier: workspace:* version: link:../database + lru-cache: + specifier: ^11.2.4 + version: 11.2.4 devDependencies: '@internal/run-ops-database': specifier: workspace:*