chore: initial project import

scripts/ops-incident.mjs

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+
+import { PrismaClient } from "@prisma/client";
+
+const prisma = new PrismaClient();
+
+const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000);
+const sixHoursAgo = new Date(Date.now() - 6 * 60 * 60 * 1000);
+const oneDayAgo = new Date(Date.now() - 24 * 60 * 60 * 1000);
+
+function formatDate(date) {
+  if (!date) {
+    return null;
+  }
+  return new Date(date).toISOString();
+}
+
+function printBlock(title, value) {
+  console.log(`\n=== ${title} ===`);
+  console.log(JSON.stringify(value, null, 2));
+}
+
+async function main() {
+  try {
+    const [retryState, webhookFailuresHour, webhookFailuresDay, failedCampaigns, disconnectedChannels, retryQueued] =
+      await Promise.all([
+        prisma.backgroundJobState.findUnique({
+          where: { jobName: "campaign-retry-worker" },
+          select: {
+            jobName: true,
+            lockedUntil: true,
+            lastRunStartedAt: true,
+            lastRunCompletedAt: true,
+            lastRunStatus: true,
+            consecutiveFailures: true,
+            lastError: true
+          }
+        }),
+        prisma.webhookEvent.count({ where: { processStatus: "failed", createdAt: { gte: oneHourAgo } } }),
+        prisma.webhookEvent.count({ where: { processStatus: "failed", createdAt: { gte: oneDayAgo } } }),
+        prisma.broadcastCampaign.count({ where: { status: { in: ["PROCESSING", "SCHEDULED"] } }),
+        prisma.channel.count({ where: { status: "DISCONNECTED" } }),
+        prisma.campaignRecipient.count({ where: { sendStatus: { in: ["QUEUED", "FAILED"] } })
+      ]);
+
+    const pendingRetry = await prisma.campaignRecipient.aggregate({
+      _count: { _all: true },
+      where: {
+        OR: [{ sendStatus: "QUEUED" }, { sendStatus: "FAILED" }]
+      }
+    });
+
+    const recentWebhook = await prisma.webhookEvent.findMany({
+      where: { processStatus: "failed", createdAt: { gte: oneHourAgo } },
+      orderBy: { createdAt: "desc" },
+      take: 10,
+      select: { id: true, tenantId: true, channelId: true, eventType: true, createdAt: true, providerEventId: true, failedReason: true }
+    });
+
+    printBlock("Health Snapshot", {
+      runAt: new Date().toISOString(),
+      retryWorker: {
+        ...retryState,
+        lockedUntil: formatDate(retryState?.lockedUntil),
+        lastRunStartedAt: formatDate(retryState?.lastRunStartedAt),
+        lastRunCompletedAt: formatDate(retryState?.lastRunCompletedAt)
+      },
+      failedCampaignsInProgress: failedCampaigns,
+      disconnectedChannels,
+      pendingRetryCandidates: pendingRetry._count._all,
+      webhookFailuresHour,
+      webhookFailuresDay
+    });
+
+    printBlock("Recent Failed Webhooks (1h)", recentWebhook);
+    printBlock("Retry Queue", { totalCandidates: retryQueued });
+  } catch (error) {
+    console.error("incident snapshot failed:", error instanceof Error ? error.message : String(error));
+    process.exitCode = 1;
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+
+await main();