chore(webapp,run-engine): downgrade boundary log noise to warn (#3462)

## Summary

Several boundary catches and customer-input validation paths were
logging at `error` level for failures the system already handles
gracefully — disconnect on auth failure, return undefined, skip retries,
etc. This batch routes them to `warn` (which stays in stdout) or counts
them as OTel metrics, so visibility is preserved without surfacing them
as alerts.

## Changes

**New helper / pattern:**
- `apiBuilder.server.ts` — `logBoundaryError(message, error, url)`
inspects the inner error type at loader/action boundary catches;
downgrades to `warn` for `AbortError`, `ServiceValidationError`, and
`EngineServiceValidationError`.
- `platform.v3.server.ts` — `platform_client.failures_total` OTel
counter with `{function, kind}` labels; helper
`recordPlatformFailure(fn, kind)` replaces the previous error-level
logging across all `BillingClient` wrappers.

**Log-level downgrades:**
- `handleSocketIo.server.ts` — `Worker authentication failed` → warn
(system disconnects on failure; refs TRI-8863)
- `waitpointSystem.ts` — when `runStatus === "CANCELED"` in the
suspended-without-checkpoint branch, skip the throw and warn instead
(benign cancel-vs-resume race, nothing to resume)
- `runAttemptSystem.ts` — `flushedMetadata` parse/validate failures →
warn (customer-side data shape, system returns gracefully)
- `batch-queue/index.ts` — final-attempt failures with
`result.skipRetries` → warn (callbacks already opted out of retry, e.g.
queue size limit hit)
- `queryPerformanceMonitor.server.ts` — slow queries → warn
(observability signal, not an application error)
- `timeoutDeployment.server.ts` — deployment-state mismatch in the
timeout job → warn (timeout-vs-completion race)

**Inner error preservation:**
- `waitpointCompletionPacket.server.ts` — `logger.error(uploadError)`
before throwing the `ServiceValidationError` wrapper, so the underlying
upload error stays visible.

## Why

The pattern across all of these is the same: a boundary log treated any
thrown/returned error as `error` regardless of cause, even when the
cause was an expected, system-handled condition (client disconnect,
customer quota, race condition, schema validation of customer data).
That made the logs noisy and made it harder to spot real bugs.

Where the underlying signal is still useful operationally (slow queries,
billing call failures), we route it to OTel metrics with low-cardinality
labels so dashboards and alerts can be tuned independently of error
logs.

## Test plan

- [ ] `pnpm run typecheck --filter webapp`
- [ ] `pnpm run build --filter @internal/run-engine`
- [ ] Trigger a run on hello-world and verify task lifecycle is
unaffected
- [ ] Cancel a suspended run and verify the cancel-while-suspended
branch in `waitpointSystem.ts` returns `{status: "skipped"}` instead of
throwing
- [ ] Confirm `platform_client.failures_total` counter shows up in
metrics with `{function, kind}` labels when the billing client errors

Author: ericallam

apps/webapp/app/runEngine/concerns/waitpointCompletionPacket.server.ts

@@ -2,6 +2,7 @@ import { type IOPacket, packetRequiresOffloading, tryCatch } from "@trigger.dev/
 import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { env } from "~/env.server";
 import { uploadPacketToObjectStore } from "~/v3/objectStore.server";
+import { logger } from "~/services/logger.server";
 import { ServiceValidationError } from "~/v3/services/common.server";
 
 function packetExtensionForDataType(dataType: string): string {
@@ -53,6 +54,11 @@ export async function processWaitpointCompletionPacket(
   );
 
   if (uploadError) {
+    logger.error("Failed to upload large waitpoint to object store", {
+      error: uploadError,
+      filename,
+      environmentId: environment.id,
+    });
     throw new ServiceValidationError("Failed to upload large waitpoint to object store", 500);
   }
 

apps/webapp/app/services/platform.v3.server.ts

@@ -31,6 +31,7 @@ import { newProjectPath, organizationBillingPath } from "~/utils/pathBuilder";
 import { singleton } from "~/utils/singleton";
 import { RedisCacheStore } from "./unkey/redisCacheStore.server";
 import { $replica } from "~/db.server";
+import { metrics } from "@opentelemetry/api";
 
 function initializeClient() {
   if (isCloud() && process.env.BILLING_API_URL && process.env.BILLING_API_KEY) {
@@ -43,6 +44,23 @@ function initializeClient() {
 }
 
 const client = singleton("billingClient", initializeClient);
+// Failures from @trigger.dev/platform billing client calls are tracked via
+// this metric (with low-cardinality {function, kind} labels) rather than
+// logged. Every task invocation hits these paths, so per-call logs were too
+// noisy; dashboard the counter for visibility instead.
+const platformClientMeter = metrics.getMeter("trigger.dev/platform-client");
+const platformClientFailuresCounter = platformClientMeter.createCounter(
+  "platform_client.failures_total",
+  {
+    description:
+      "Failures returned or thrown by @trigger.dev/platform billing client calls",
+  }
+);
+
+function recordPlatformFailure(fn: string, kind: "caught" | "no_success") {
+  platformClientFailuresCounter.add(1, { function: fn, kind });
+}
+
 
 function initializePlatformCache() {
   const ctx = new DefaultStatefulContext();
@@ -206,7 +224,7 @@ export async function getCurrentPlan(orgId: string) {
     firstDayOfNextMonth.setUTCHours(0, 0, 0, 0);
 
     if (!result.success) {
-      logger.error("Error getting current plan - no success", { orgId, error: result.error });
+      recordPlatformFailure("getCurrentPlan", "no_success");
       return undefined;
     }
 
@@ -222,7 +240,7 @@ export async function getCurrentPlan(orgId: string) {
 
     return { ...result, usage };
   } catch (e) {
-    logger.error("Error getting current plan - caught error", { orgId, error: e });
+    recordPlatformFailure("getCurrentPlan", "caught");
     return undefined;
   }
 }
@@ -233,13 +251,13 @@ export async function getLimits(orgId: string) {
   try {
     const result = await client.currentPlan(orgId);
     if (!result.success) {
-      logger.error("Error getting limits - no success", { orgId, error: result.error });
+      recordPlatformFailure("getLimits", "no_success");
       return undefined;
     }
 
     return result.v3Subscription?.plan?.limits;
   } catch (e) {
-    logger.error("Error getting limits - caught error", { orgId, error: e });
+    recordPlatformFailure("getLimits", "caught");
     return undefined;
   }
 }
@@ -315,7 +333,7 @@ export async function customerPortalUrl(orgId: string, orgSlug: string) {
       returnUrl: `${env.APP_ORIGIN}${organizationBillingPath({ slug: orgSlug })}`,
     });
   } catch (e) {
-    logger.error("Error getting customer portal Url", { orgId, error: e });
+    recordPlatformFailure("customerPortalUrl", "caught");
     return undefined;
   }
 }
@@ -326,12 +344,12 @@ export async function getPlans() {
   try {
     const result = await client.plans();
     if (!result.success) {
-      logger.error("Error getting plans - no success", { error: result.error });
+      recordPlatformFailure("getPlans", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting plans - caught error", { error: e });
+    recordPlatformFailure("getPlans", "caught");
     return undefined;
   }
 }
@@ -408,12 +426,12 @@ export async function setConcurrencyAddOn(organizationId: string, amount: number
   try {
     const result = await client.setAddOn(organizationId, { type: "concurrency", amount });
     if (!result.success) {
-      logger.error("Error setting concurrency add on - no success", { error: result.error });
+      recordPlatformFailure("setConcurrencyAddOn", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error setting concurrency add on - caught error", { error: e });
+    recordPlatformFailure("setConcurrencyAddOn", "caught");
     return undefined;
   }
 }
@@ -424,12 +442,12 @@ export async function setSeatsAddOn(organizationId: string, amount: number) {
   try {
     const result = await client.setAddOn(organizationId, { type: "seats", amount });
     if (!result.success) {
-      logger.error("Error setting seats add on - no success", { error: result.error });
+      recordPlatformFailure("setSeatsAddOn", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error setting seats add on - caught error", { error: e });
+    recordPlatformFailure("setSeatsAddOn", "caught");
     return undefined;
   }
 }
@@ -440,12 +458,12 @@ export async function setBranchesAddOn(organizationId: string, amount: number) {
   try {
     const result = await client.setAddOn(organizationId, { type: "branches", amount });
     if (!result.success) {
-      logger.error("Error setting branches add on - no success", { error: result.error });
+      recordPlatformFailure("setBranchesAddOn", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error setting branches add on - caught error", { error: e });
+    recordPlatformFailure("setBranchesAddOn", "caught");
     return undefined;
   }
 }
@@ -456,12 +474,12 @@ export async function getUsage(organizationId: string, { from, to }: { from: Dat
   try {
     const result = await client.usage(organizationId, { from, to });
     if (!result.success) {
-      logger.error("Error getting usage - no success", { error: result.error });
+      recordPlatformFailure("getUsage", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting usage - caught error", { error: e });
+    recordPlatformFailure("getUsage", "caught");
     return undefined;
   }
 }
@@ -490,12 +508,12 @@ export async function getUsageSeries(organizationId: string, params: UsageSeries
   try {
     const result = await client.usageSeries(organizationId, params);
     if (!result.success) {
-      logger.error("Error getting usage series - no success", { error: result.error });
+      recordPlatformFailure("getUsageSeries", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error getting usage series - caught error", { error: e });
+    recordPlatformFailure("getUsageSeries", "caught");
     return undefined;
   }
 }
@@ -514,12 +532,12 @@ export async function reportInvocationUsage(
       additionalData,
     });
     if (!result.success) {
-      logger.error("Error reporting invocation - no success", { error: result.error });
+      recordPlatformFailure("reportInvocationUsage", "no_success");
       return undefined;
     }
     return result;
   } catch (e) {
-    logger.error("Error reporting invocation - caught error", { error: e });
+    recordPlatformFailure("reportInvocationUsage", "caught");
     return undefined;
   }
 }
@@ -550,12 +568,12 @@ export async function getEntitlement(
     try {
       const response = await client.getEntitlement(organizationId);
       if (!response.success) {
-        logger.error("Error getting entitlement - no success", { error: response.error });
+        recordPlatformFailure("getEntitlement", "no_success");
         return undefined;
       }
       return response;
     } catch (e) {
-      logger.error("Error getting entitlement - caught error", { error: e });
+      recordPlatformFailure("getEntitlement", "caught");
       return undefined;
     }
   });
@@ -602,7 +620,7 @@ export async function getBillingAlerts(
   if (!client) return undefined;
   const result = await client.getBillingAlerts(organizationId);
   if (!result.success) {
-    logger.error("Error getting billing alert", { error: result.error, organizationId });
+    recordPlatformFailure("getBillingAlert", "no_success");
     throw new Error("Error getting billing alert");
   }
   return result;
@@ -615,7 +633,7 @@ export async function setBillingAlert(
   if (!client) return undefined;
   const result = await client.updateBillingAlerts(organizationId, alert);
   if (!result.success) {
-    logger.error("Error setting billing alert", { error: result.error, organizationId });
+    recordPlatformFailure("setBillingAlert", "no_success");
     throw new Error("Error setting billing alert");
   }
   return result;
@@ -628,11 +646,7 @@ export async function generateRegistryCredentials(
   if (!client) return undefined;
   const result = await client.generateRegistryCredentials(projectId, region);
   if (!result.success) {
-    logger.error("Error generating registry credentials", {
-      error: result.error,
-      projectId,
-      region,
-    });
+    recordPlatformFailure("generateRegistryCredentials", "no_success");
     throw new Error("Failed to generate registry credentials");
   }
 
@@ -651,13 +665,7 @@ export async function enqueueBuild(
   if (!client) return undefined;
   const result = await client.enqueueBuild(projectId, { deploymentId, artifactKey, options });
   if (!result.success) {
-    logger.error("Error enqueuing build", {
-      error: result.error,
-      projectId,
-      deploymentId,
-      artifactKey,
-      options,
-    });
+    recordPlatformFailure("enqueueBuild", "no_success");
     throw new Error("Failed to enqueue build");
   }
 
@@ -672,12 +680,12 @@ export async function getPrivateLinks(
   const [error, result] = await tryCatch(client.getPrivateLinks(organizationId));
 
   if (error) {
-    logger.error("Error getting private links", { organizationId, error });
+    recordPlatformFailure("getPrivateLinks", "caught");
     return undefined;
   }
 
   if (!result.success) {
-    logger.error("Error getting private links - no success", { organizationId, error: result.error });
+    recordPlatformFailure("getPrivateLinks", "no_success");
     return undefined;
   }
 
@@ -693,12 +701,12 @@ export async function createPrivateLink(
   const [error, result] = await tryCatch(client.createPrivateLink(organizationId, body));
 
   if (error) {
-    logger.error("Error creating private link", { organizationId, error });
+    recordPlatformFailure("createPrivateLink", "caught");
     throw error;
   }
 
   if (!result.success) {
-    logger.error("Error creating private link - no success", { organizationId, error: result.error });
+    recordPlatformFailure("createPrivateLink", "no_success");
     throw new Error(result.error ?? "Failed to create private link");
   }
 
@@ -714,12 +722,12 @@ export async function deletePrivateLink(
   const [error, result] = await tryCatch(client.deletePrivateLink(organizationId, connectionId));
 
   if (error) {
-    logger.error("Error deleting private link", { organizationId, connectionId, error });
+    recordPlatformFailure("deletePrivateLink", "caught");
     throw error;
   }
 
   if (!result.success) {
-    logger.error("Error deleting private link - no success", { organizationId, connectionId, error: result.error });
+    recordPlatformFailure("deletePrivateLink", "no_success");
     throw new Error(result.error ?? "Failed to delete private link");
   }
 }
@@ -732,12 +740,12 @@ export async function getPrivateLinkRegions(
   const [error, result] = await tryCatch(client.getPrivateLinkRegions(organizationId));
 
   if (error) {
-    logger.error("Error getting private link regions", { organizationId, error });
+    recordPlatformFailure("getPrivateLinkRegions", "caught");
     return undefined;
   }
 
   if (!result.success) {
-    logger.error("Error getting private link regions - no success", { organizationId, error: result.error });
+    recordPlatformFailure("getPrivateLinkRegions", "no_success");
     return undefined;
   }