feat: incident detection automation

apps/api/prisma/migrations/20260223044415_alter_table_pull_requests_add_columns/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "PullRequest" ADD COLUMN     "body" TEXT NOT NULL DEFAULT '',
+ADD COLUMN     "labels" JSONB NOT NULL DEFAULT '[]',
+ADD COLUMN     "sourceBranch" TEXT NOT NULL DEFAULT '';

apps/api/prisma/schema.prisma

@@ -369,8 +369,11 @@ model PullRequest {
 
   title        String
   number       String
+  sourceBranch String @default("")
   targetBranch String @default("main")
+  body         String @default("")
 
+  labels            Json @default("[]")
   files             Json @default("[]")
   commentCount      Int
   changedFilesCount Int

apps/api/src/app/activity-events/services/work-log.unit.test.ts

@@ -58,12 +58,14 @@ describe("groupSerialReviews", () => {
     type,
     eventAt,
     pullRequest: {
+      sourceBranch: "feat/test",
       targetBranch: "main",
       mergeCommitSha: null,
       gitProvider: "GITHUB",
       gitPullRequestId: "1",
       gitUrl: "https://github.com/test/test/pull/1",
       title: "Test PR",
+      body: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
       number: "1",
       files: [],
       commentCount: 0,
@@ -78,6 +80,7 @@ describe("groupSerialReviews", () => {
       authorId: John,
       repositoryId: 1,
       workspaceId: 1,
+      labels: [],
       ...pullRequest,
     },
   });

apps/api/src/app/deployment/services/deployment-create-from-merge.service.ts

@@ -9,6 +9,7 @@ import {
 } from "./deployment-pr-linking.service";
 import { CreateDeploymentFromPullRequestMergeArgs } from "./deployment-create-from-merge.types";
 import { BusinessRuleException } from "../../errors/exceptions/business-rule.exception";
+import { addJob, SweetQueue } from "../../../bull-mq/queues";
 
 export const createDeploymentFromPullRequestMerge = async ({
   application,
@@ -79,6 +80,11 @@ export const createDeploymentFromPullRequestMerge = async ({
     workspaceId,
   });
 
+  await addJob(SweetQueue.AUTOMATION_INCIDENT_DETECTION, {
+    deploymentId: deployment.id,
+    workspaceId,
+  });
+
   logger.info("deploymentCreateFromMergeWorker: Deployment created", {
     deployment,
   });

apps/api/src/app/deployment/services/deployment-pr-linking.service.ts

@@ -25,6 +25,7 @@ import { getTimeToDeploy } from "../../github/services/github-pull-request-track
 import { DataIntegrityException } from "../../errors/exceptions/data-integrity.exception";
 import { isBefore } from "date-fns";
 import { captureException } from "../../../lib/sentry";
+import { addJob, SweetQueue } from "../../../bull-mq/queues";
 
 export const handleDeploymentPullRequestAutoLinking = async ({
   workspaceId,
@@ -158,6 +159,11 @@ export const handleDeploymentPullRequestAutoLinking = async ({
     pullRequestIds: filteredPullRequests.map((pr) => pr.id),
   });
 
+  await addJob(SweetQueue.AUTOMATION_INCIDENT_DETECTION, {
+    deploymentId,
+    workspaceId,
+  });
+
   await Promise.all(
     filteredPullRequests.map(async (pr) => {
       if (!pr.tracking) {

apps/api/src/app/github/services/github-pull-request.service.ts

@@ -124,6 +124,14 @@ const fetchPullRequest = async (
             closedAt
             mergedAt
             baseRefName
+            headRefName
+            body
+
+            labels(first: 100) {
+              nodes {
+                name
+              }
+            }
 
             createdAt
             updatedAt
@@ -287,8 +295,11 @@ const upsertPullRequest = async (
     gitPullRequestId: gitPrData.id,
     gitUrl: gitPrData.url,
     title: gitPrData.title,
+    sourceBranch: gitPrData.headRefName ?? "",
     targetBranch: gitPrData.baseRefName,
+    body: gitPrData.body ?? "",
     number: gitPrData.number.toString(),
+    labels: gitPrData.labels?.nodes?.map((l: { name: string }) => l.name) ?? [],
     commentCount: gitPrData.totalCommentsCount,
     changedFilesCount: gitPrData.changedFiles,
     linesAddedCount: gitPrData.additions,

apps/api/src/app/incidents/services/incident-detection.service.ts

@@ -0,0 +1,256 @@
+import { AutomationType, PullRequest } from "@prisma/client";
+import { getPrisma } from "../../../prisma";
+import { logger } from "../../../lib/logger";
+import { ResourceNotFoundException } from "../../errors/exceptions/resource-not-found.exception";
+import { findDeploymentByIdOrThrow } from "../../deployment/services/deployment.service";
+import { findWorkspaceById } from "../../workspaces/services/workspace.service";
+import { findAutomationByType } from "../../automations/services/automation.service";
+import { isActiveCustomer } from "../../authorization.service";
+import { IncidentDetectionSettings } from "../../automations/services/automation.types";
+import { HandleIncidentDetectionAutomationArgs } from "./incident-detection.types";
+
+interface DetectionResult {
+  causeDeploymentId: number;
+  fixDeploymentId: number;
+}
+
+export const handleIncidentDetectionAutomation = async ({
+  workspaceId,
+  deploymentId,
+}: HandleIncidentDetectionAutomationArgs) => {
+  const workspace = await findWorkspaceById(workspaceId);
+
+  if (!workspace) {
+    throw new ResourceNotFoundException(
+      "handleDeploymentIncidentDetection: Workspace not found",
+      { extra: { workspaceId } }
+    );
+  }
+
+  const automation = await findAutomationByType({
+    workspaceId,
+    type: AutomationType.INCIDENT_DETECTION,
+  });
+
+  if (!automation?.enabled) return;
+
+  if (!isActiveCustomer(workspace)) return;
+
+  const settings = automation.settings as IncidentDetectionSettings;
+
+  const deployment = await findDeploymentByIdOrThrow(
+    { workspaceId, deploymentId },
+    {
+      include: {
+        pullRequests: {
+          include: {
+            pullRequest: true,
+          },
+        },
+      },
+    }
+  );
+
+  const pullRequests = deployment.pullRequests.map((dp) => dp.pullRequest);
+
+  const result =
+    (await detectRollback(settings, deployment, workspaceId)) ??
+    (await detectRevert(settings, pullRequests, deployment, workspaceId)) ??
+    (await detectHotfix(settings, pullRequests, deployment, workspaceId));
+
+  if (!result) return;
+
+  const existingIncident = await getPrisma(workspaceId).incident.findFirst({
+    where: { causeDeploymentId: result.causeDeploymentId, workspaceId },
+  });
+
+  if (existingIncident) {
+    logger.info(
+      "handleDeploymentIncidentDetection: Incident already exists for cause deployment",
+      { causeDeploymentId: result.causeDeploymentId, existingIncident }
+    );
+    return;
+  }
+
+  await getPrisma(workspaceId).incident.create({
+    data: {
+      causeDeploymentId: result.causeDeploymentId,
+      fixDeploymentId: result.fixDeploymentId,
+      detectedAt: deployment.deployedAt,
+      workspaceId,
+    },
+  });
+
+  logger.info("handleDeploymentIncidentDetection: Incident created", {
+    deploymentId,
+    causeDeploymentId: result.causeDeploymentId,
+    fixDeploymentId: result.fixDeploymentId,
+  });
+};
+
+const detectRollback = async (
+  settings: IncidentDetectionSettings,
+  deployment: {
+    id: number;
+    version: string;
+    applicationId: number;
+    environmentId: number;
+    deployedAt: Date;
+  },
+  workspaceId: number
+): Promise<DetectionResult | null> => {
+  if (!settings.rollback?.enabled) return null;
+
+  const rolledBackTo = await getPrisma(workspaceId).deployment.findFirst({
+    where: {
+      workspaceId,
+      applicationId: deployment.applicationId,
+      environmentId: deployment.environmentId,
+      version: deployment.version,
+      deployedAt: { lt: deployment.deployedAt },
+      id: { not: deployment.id },
+      archivedAt: null,
+    },
+    orderBy: { deployedAt: "desc" },
+  });
+
+  if (!rolledBackTo) return null;
+
+  const causeDeployment = await getPrisma(workspaceId).deployment.findFirst({
+    where: {
+      workspaceId,
+      applicationId: deployment.applicationId,
+      environmentId: deployment.environmentId,
+      deployedAt: { gt: rolledBackTo.deployedAt },
+      id: { not: deployment.id },
+      archivedAt: null,
+    },
+    orderBy: { deployedAt: "asc" },
+  });
+
+  if (!causeDeployment) return null;
+
+  logger.info("detectRollback: Rollback detected", {
+    deploymentId: deployment.id,
+    rolledBackToId: rolledBackTo.id,
+    causeDeploymentId: causeDeployment.id,
+  });
+
+  return {
+    causeDeploymentId: causeDeployment.id,
+    fixDeploymentId: deployment.id,
+  };
+};
+
+const detectRevert = async (
+  settings: IncidentDetectionSettings,
+  pullRequests: PullRequest[],
+  deployment: { id: number; applicationId: number; environmentId: number },
+  workspaceId: number
+): Promise<DetectionResult | null> => {
+  if (!settings.revert?.enabled) return null;
+
+  const revertPattern = /^Revert "(.+)"$/;
+
+  for (const pr of pullRequests) {
+    const match = pr.title.match(revertPattern);
+    if (!match) continue;
+
+    const originalTitle = match[1];
+
+    const originalPr = await getPrisma(workspaceId).pullRequest.findFirst({
+      where: {
+        title: originalTitle,
+        repositoryId: pr.repositoryId,
+        mergedAt: { not: null },
+        id: { not: pr.id },
+      },
+      orderBy: { mergedAt: "desc" },
+      include: {
+        deploymentEvents: true,
+      },
+    });
+
+    if (!originalPr) continue;
+
+    const deploymentLink = originalPr.deploymentEvents.find(
+      (de) => de.deploymentId !== deployment.id
+    );
+
+    if (!deploymentLink) continue;
+
+    logger.info("detectRevert: Revert detected", {
+      revertPrId: pr.id,
+      originalPrId: originalPr.id,
+      causeDeploymentId: deploymentLink.deploymentId,
+    });
+
+    return {
+      causeDeploymentId: deploymentLink.deploymentId,
+      fixDeploymentId: deployment.id,
+    };
+  }
+
+  return null;
+};
+
+const detectHotfix = async (
+  settings: IncidentDetectionSettings,
+  pullRequests: PullRequest[],
+  deployment: {
+    id: number;
+    applicationId: number;
+    environmentId: number;
+    deployedAt: Date;
+  },
+  workspaceId: number
+): Promise<DetectionResult | null> => {
+  if (!settings.hotfix?.enabled) return null;
+
+  const { prTitleRegEx, branchRegEx, prLabelRegEx } = settings.hotfix;
+
+  const isHotfix = pullRequests.some((pr) => {
+    if (prTitleRegEx && new RegExp(prTitleRegEx, "i").test(pr.title)) {
+      return true;
+    }
+
+    if (branchRegEx && new RegExp(branchRegEx, "i").test(pr.sourceBranch)) {
+      return true;
+    }
+
+    if (prLabelRegEx) {
+      const labels = (pr.labels as string[]) ?? [];
+      if (labels.some((label) => new RegExp(prLabelRegEx, "i").test(label))) {
+        return true;
+      }
+    }
+
+    return false;
+  });
+
+  if (!isHotfix) return null;
+
+  const previousDeployment = await getPrisma(workspaceId).deployment.findFirst({
+    where: {
+      workspaceId,
+      applicationId: deployment.applicationId,
+      environmentId: deployment.environmentId,
+      deployedAt: { lt: deployment.deployedAt },
+      id: { not: deployment.id },
+      archivedAt: null,
+    },
+    orderBy: { deployedAt: "desc" },
+  });
+
+  if (!previousDeployment) return null;
+
+  logger.info("detectHotfix: Hotfix detected", {
+    deploymentId: deployment.id,
+    causeDeploymentId: previousDeployment.id,
+  });
+
+  return {
+    causeDeploymentId: previousDeployment.id,
+    fixDeploymentId: deployment.id,
+  };
+};

apps/api/src/app/incidents/services/incident-detection.types.ts

@@ -0,0 +1,4 @@
+export interface HandleIncidentDetectionAutomationArgs {
+  workspaceId: number;
+  deploymentId: number;
+}

apps/api/src/app/incidents/workers/automation-incident-detection.worker.ts

@@ -0,0 +1,22 @@
+import { Job } from "bullmq";
+import { SweetQueue } from "../../../bull-mq/queues";
+import { createWorker } from "../../../bull-mq/workers";
+import { logger } from "../../../lib/logger";
+import { handleIncidentDetectionAutomation } from "../services/incident-detection.service";
+
+interface AutomationIncidentDetectionJobData {
+  deploymentId: number;
+  workspaceId: number;
+}
+
+export const automationIncidentDetectionWorker = createWorker(
+  SweetQueue.AUTOMATION_INCIDENT_DETECTION,
+  async (job: Job<AutomationIncidentDetectionJobData>) => {
+    logger.info("[AUTOMATION_INCIDENT_DETECTION]", { data: job.data });
+
+    await handleIncidentDetectionAutomation({
+      workspaceId: job.data.workspaceId,
+      deploymentId: job.data.deploymentId,
+    });
+  }
+);

apps/api/src/bull-mq/queues.ts

@@ -37,6 +37,7 @@ export enum SweetQueue {
   // Automations
   AUTOMATION_PR_TITLE_CHECK = "{automation.pr_title_check}",
   AUTOMATION_PR_SIZE_LABELER = "{automation.pr_size_labeler}",
+  AUTOMATION_INCIDENT_DETECTION = "{automation.incident_detection}",
 
   // Alerts
   ALERT_MERGED_WITHOUT_APPROVAL = "{alert.merged_without_approval}",