Enhance API with OIDC support, including login and callback endpoints. Update environment variables for OIDC configuration in .env.example. Add new features to the catalog service for listing software families, Minecraft versions, and deployment regions. Implement server management actions such as kill, delete, and update in the servers module. Integrate feature flags for maintenance mode in server operations. Update pnpm-lock.yaml with new dependencies and versions.

apps/worker/package.json

@@ -16,9 +16,13 @@
     "@hexahost/config": "workspace:*",
     "@hexahost/database": "workspace:*",
     "@hexahost/dns": "workspace:*",
+    "@hexahost/email-templates": "workspace:*",
     "@hexahost/metering": "workspace:*",
     "@hexahost/scheduler": "workspace:*",
+    "@hexahost/server-state": "workspace:*",
     "@hexahost/storage": "workspace:*",
+    "@aws-sdk/client-s3": "^3.758.0",
+    "cron-parser": "^5.0.4",
     "bullmq": "^5.34.10",
     "ioredis": "^5.4.2",
     "nodemailer": "^6.10.0",

apps/worker/src/config.ts

@@ -1,7 +1,13 @@
 import { z } from 'zod';
 
 export const notificationEmailJobSchema = z.object({
-  type: z.enum(['email_verification', 'password_reset']),
+  type: z.enum([
+    'email_verification',
+    'password_reset',
+    'server_ready',
+    'backup_success',
+    'backup_failed',
+  ]),
   to: z.string().email(),
   subject: z.string(),
   template: z.string(),

apps/worker/src/handlers/notification-dispatch.ts

@@ -0,0 +1,106 @@
+import { Job } from 'bullmq';
+import { z } from 'zod';
+
+import { renderBackupFailedEmail, renderBackupSuccessEmail, renderServerReadyEmail } from '@hexahost/email-templates';
+
+import { getSmtpConfig } from '../config';
+import { sendEmail } from '../email';
+import { logger } from '../logger';
+
+const dispatchJobSchema = z.object({
+  userId: z.string().uuid(),
+  type: z.string(),
+  channel: z.enum(['EMAIL', 'WEBHOOK', 'DISCORD']),
+  title: z.string(),
+  body: z.string(),
+  metadata: z.record(z.unknown()).optional(),
+});
+
+async function postWebhook(url: string, payload: Record<string, unknown>): Promise<void> {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+  if (!response.ok) {
+    throw new Error(`Webhook delivery failed: ${response.status}`);
+  }
+}
+
+async function postDiscord(webhookUrl: string, content: string): Promise<void> {
+  const response = await fetch(webhookUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ content: content.slice(0, 1900) }),
+  });
+  if (!response.ok) {
+    throw new Error(`Discord delivery failed: ${response.status}`);
+  }
+}
+
+export async function processNotificationDispatchJob(job: Job): Promise<{ delivered: boolean }> {
+  if (job.name !== 'dispatch-notification') {
+    return { delivered: false };
+  }
+
+  const payload = dispatchJobSchema.parse(job.data);
+  const config = getSmtpConfig();
+  const locale = payload.metadata?.['locale'] === 'de' ? 'de' : 'en';
+  const serverName = String(payload.metadata?.['serverName'] ?? 'Server');
+
+  if (payload.channel === 'EMAIL') {
+    const userEmail = payload.metadata?.['email'];
+    if (typeof userEmail !== 'string') {
+      logger.warn({ jobId: job.id }, 'Email notification missing recipient');
+      return { delivered: false };
+    }
+
+    let subject = payload.title;
+    let html = `<p>${payload.body}</p>`;
+
+    if (payload.type === 'SERVER_READY') {
+      const rendered = renderServerReadyEmail({ appName: config.APP_NAME, serverName, locale });
+      subject = rendered.subject;
+      html = rendered.html;
+    } else if (payload.type === 'BACKUP_SUCCESS') {
+      const rendered = renderBackupSuccessEmail({ appName: config.APP_NAME, serverName, locale });
+      subject = rendered.subject;
+      html = rendered.html;
+    } else if (payload.type === 'BACKUP_FAILED') {
+      const rendered = renderBackupFailedEmail({ appName: config.APP_NAME, serverName, locale });
+      subject = rendered.subject;
+      html = rendered.html;
+    }
+
+    await sendEmail({ to: userEmail, subject, html });
+    return { delivered: true };
+  }
+
+  if (payload.channel === 'WEBHOOK') {
+    const url = process.env['NOTIFICATION_WEBHOOK_URL'];
+    if (!url) {
+      logger.warn('NOTIFICATION_WEBHOOK_URL not configured');
+      return { delivered: false };
+    }
+    await postWebhook(url, {
+      userId: payload.userId,
+      type: payload.type,
+      title: payload.title,
+      body: payload.body,
+      metadata: payload.metadata ?? {},
+    });
+    return { delivered: true };
+  }
+
+  if (payload.channel === 'DISCORD') {
+    const url = process.env['NOTIFICATION_DISCORD_WEBHOOK_URL'];
+    if (!url) {
+      logger.warn('NOTIFICATION_DISCORD_WEBHOOK_URL not configured');
+      return { delivered: false };
+    }
+    await postDiscord(url, `**${payload.title}**\n${payload.body}`);
+    return { delivered: true };
+  }
+
+  return { delivered: false };
+}

apps/worker/src/handlers/notifications.ts

@@ -1,9 +1,22 @@
 import { Job } from 'bullmq';
 
+import {
+  renderBackupFailedEmail,
+  renderBackupSuccessEmail,
+  renderResetEmail,
+  renderServerReadyEmail,
+  renderVerifyEmail,
+  type EmailLocale,
+} from '@hexahost/email-templates';
+
 import { getSmtpConfig, notificationEmailJobSchema } from '../config';
 import { sendEmail } from '../email';
 import { logger } from '../logger';
 
+function localeFromData(data: Record<string, unknown>): EmailLocale {
+  return data['locale'] === 'de' ? 'de' : 'en';
+}
+
 export async function processNotificationJob(job: Job): Promise<{ sent: boolean }> {
   if (job.name !== 'send-email') {
     logger.warn({ jobName: job.name }, 'Unknown notification job');
@@ -12,19 +25,58 @@ export async function processNotificationJob(job: Job): Promise<{ sent: boolean
 
   const payload = notificationEmailJobSchema.parse(job.data);
   const config = getSmtpConfig();
+  const locale = localeFromData(payload.data);
 
+  let subject = payload.subject;
   let html: string;
+
   if (payload.type === 'email_verification') {
-    const verifyUrl = String(payload.data['verifyUrl'] ?? '');
-    html = `<p>Welcome to ${config.APP_NAME}!</p><p><a href="${verifyUrl}">Verify your email address</a></p><p>This link expires in 24 hours.</p>`;
+    const rendered = renderVerifyEmail({
+      appName: config.APP_NAME,
+      url: String(payload.data['verifyUrl'] ?? ''),
+      locale,
+    });
+    subject = rendered.subject;
+    html = rendered.html;
+  } else if (payload.type === 'password_reset') {
+    const rendered = renderResetEmail({
+      appName: config.APP_NAME,
+      url: String(payload.data['resetUrl'] ?? ''),
+      locale,
+    });
+    subject = rendered.subject;
+    html = rendered.html;
+  } else if (payload.type === 'server_ready') {
+    const rendered = renderServerReadyEmail({
+      appName: config.APP_NAME,
+      serverName: String(payload.data['serverName'] ?? 'Server'),
+      locale,
+    });
+    subject = rendered.subject;
+    html = rendered.html;
+  } else if (payload.type === 'backup_success') {
+    const rendered = renderBackupSuccessEmail({
+      appName: config.APP_NAME,
+      serverName: String(payload.data['serverName'] ?? 'Server'),
+      locale,
+    });
+    subject = rendered.subject;
+    html = rendered.html;
+  } else if (payload.type === 'backup_failed') {
+    const rendered = renderBackupFailedEmail({
+      appName: config.APP_NAME,
+      serverName: String(payload.data['serverName'] ?? 'Server'),
+      locale,
+    });
+    subject = rendered.subject;
+    html = rendered.html;
   } else {
-    const resetUrl = String(payload.data['resetUrl'] ?? '');
-    html = `<p>You requested a password reset for ${config.APP_NAME}.</p><p><a href="${resetUrl}">Set a new password</a></p><p>If you did not request this, ignore this email.</p>`;
+    html = `<p>${payload.subject}</p>`;
   }
 
   await sendEmail({
     to: payload.to,
-    subject: payload.subject,
+    subject,
     html,
   });
 

apps/worker/src/handlers/schedules-compliance.ts

@@ -0,0 +1,392 @@
+import { randomUUID } from 'node:crypto';
+
+import { PutObjectCommand } from '@aws-sdk/client-s3';
+import { Queue } from 'bullmq';
+import Redis from 'ioredis';
+import { CronExpressionParser } from 'cron-parser';
+
+import { searchModrinthProjects } from '@hexahost/catalog';
+import { getConfig } from '@hexahost/config';
+import { prisma } from '@hexahost/database';
+import type { GameServerStatus } from '@hexahost/server-state';
+import { ALLOWED_TRANSITIONS } from '@hexahost/server-state';
+import {
+  createS3Client,
+  loadStorageConfig,
+} from '@hexahost/storage';
+
+import { logger } from '../logger';
+import { getBackupsQueue, getLifecycleQueue } from '../queues';
+
+export async function processScheduleTick(): Promise<void> {
+  const now = new Date();
+  const due = await prisma.schedule.findMany({
+    where: {
+      enabled: true,
+      nextRunAt: { lte: now },
+    },
+    include: { tasks: true, server: true },
+    take: 50,
+  });
+
+  if (due.length === 0) {
+    return;
+  }
+
+  const lifecycleQueue = getLifecycleQueue();
+
+  for (const schedule of due) {
+    for (const task of schedule.tasks) {
+      const correlationId = `${schedule.id}:${task.id}:${now.getTime()}`;
+      if (task.taskType === 'START') {
+        await lifecycleQueue.add('start-server', {
+          serverId: schedule.serverId,
+          action: 'start',
+          correlationId,
+        });
+      } else if (task.taskType === 'STOP') {
+        await lifecycleQueue.add('stop-server', {
+          serverId: schedule.serverId,
+          action: 'stop',
+          correlationId,
+        });
+      } else if (task.taskType === 'RESTART') {
+        await lifecycleQueue.add('restart-server', {
+          serverId: schedule.serverId,
+          action: 'restart',
+          correlationId,
+        });
+      } else if (task.taskType === 'BACKUP') {
+        const backupsQueue = getBackupsQueue();
+        await backupsQueue.add('create-backup', {
+          serverId: schedule.serverId,
+          type: 'SCHEDULED',
+        });
+      } else if (task.taskType === 'CONSOLE_COMMAND') {
+        const command =
+          typeof task.payload === 'object' &&
+          task.payload !== null &&
+          'command' in task.payload &&
+          typeof (task.payload as { command: unknown }).command === 'string'
+            ? (task.payload as { command: string }).command
+            : null;
+        if (command) {
+          await lifecycleQueue.add('console-command', {
+            serverId: schedule.serverId,
+            command,
+            correlationId,
+          });
+        }
+      }
+    }
+
+    const nextRunAt = computeNextScheduleRun(schedule.cronExpr, schedule.timezone);
+    await prisma.schedule.update({
+      where: { id: schedule.id },
+      data: { lastRunAt: now, nextRunAt },
+    });
+  }
+
+  logger.info({ count: due.length }, 'Processed due schedules');
+}
+
+function computeNextScheduleRun(cronExpr: string, timezone: string): Date {
+  try {
+    return CronExpressionParser.parse(cronExpr, { tz: timezone, currentDate: new Date() })
+      .next()
+      .toDate();
+  } catch {
+    const fallback = new Date();
+    fallback.setMinutes(fallback.getMinutes() + 5);
+    return fallback;
+  }
+}
+
+export async function processCatalogSyncTick(): Promise<void> {
+  const versions = ['1.21.1', '1.21', '1.20.6', '1.20.4'];
+  for (const version of versions) {
+    await prisma.minecraftVersion.upsert({
+      where: { version },
+      create: { version, edition: 'JAVA', isActive: true },
+      update: { isActive: true },
+    });
+  }
+
+  await prisma.catalogProvider.upsert({
+    where: { kind: 'MODRINTH' },
+    create: { kind: 'MODRINTH', name: 'Modrinth', isEnabled: true },
+    update: { isEnabled: true, lastSyncAt: new Date() },
+  });
+
+  try {
+    const result = await searchModrinthProjects({
+      query: 'fabric api',
+      projectType: 'mod',
+      gameVersion: '1.21.1',
+      loader: 'fabric',
+      limit: 5,
+    });
+    const provider = await prisma.catalogProvider.findUnique({ where: { kind: 'MODRINTH' } });
+    if (provider) {
+      for (const hit of result.hits.slice(0, 5)) {
+        await prisma.catalogProject.upsert({
+          where: {
+            providerId_externalId: {
+              providerId: provider.id,
+              externalId: hit.project_id,
+            },
+          },
+          create: {
+            providerId: provider.id,
+            externalId: hit.project_id,
+            slug: hit.slug,
+            name: hit.title,
+            projectType: hit.project_type,
+            metadata: { downloads: hit.downloads },
+          },
+          update: {
+            slug: hit.slug,
+            name: hit.title,
+            syncedAt: new Date(),
+          },
+        });
+      }
+    }
+  } catch (error) {
+    logger.warn({ error }, 'Modrinth catalog sync skipped');
+  }
+
+  logger.info({ versions: versions.length }, 'Catalog sync tick completed');
+}
+
+const EXPORT_TTL_DAYS = 7;
+const EXPORT_BLOB_PREFIX = 'export:blob:';
+
+function isStorageConfigured(): boolean {
+  return Boolean(process.env['S3_ENDPOINT']?.trim());
+}
+
+function exportObjectKey(userId: string, exportId: string): string {
+  return `users/${userId}/exports/${exportId}.json`;
+}
+
+export async function processComplianceTick(): Promise<void> {
+  const config = getConfig();
+  const redis = new Redis(config.REDIS_URL, { maxRetriesPerRequest: null });
+  const lifecycleQueue = new Queue('server-lifecycle', {
+    connection: { url: config.REDIS_URL, maxRetriesPerRequest: null },
+  });
+
+  try {
+    const exports = await prisma.dataExportRequest.findMany({
+      where: { status: 'PENDING' },
+      take: 10,
+    });
+
+    for (const row of exports) {
+      await processDataExport(row.id, redis);
+    }
+
+    const deletions = await prisma.accountDeletionRequest.findMany({
+      where: { status: 'SCHEDULED', scheduledFor: { lte: new Date() } },
+      take: 10,
+    });
+
+    for (const row of deletions) {
+      await executeAccountDeletion(row.id, lifecycleQueue);
+    }
+
+    logger.info(
+      { exports: exports.length, deletions: deletions.length },
+      'Compliance tick completed',
+    );
+  } finally {
+    await lifecycleQueue.close();
+    redis.disconnect();
+  }
+}
+
+async function processDataExport(exportId: string, redis: Redis): Promise<void> {
+  const row = await prisma.dataExportRequest.findUnique({ where: { id: exportId } });
+  if (!row || row.status !== 'PENDING') {
+    return;
+  }
+
+  await prisma.dataExportRequest.update({
+    where: { id: exportId },
+    data: { status: 'PROCESSING' },
+  });
+
+  try {
+    const payload = await collectUserExportData(row.userId);
+    const json = JSON.stringify(payload, null, 2);
+    const expiresAt = new Date();
+    expiresAt.setDate(expiresAt.getDate() + EXPORT_TTL_DAYS);
+
+    let downloadUrl: string;
+
+    if (isStorageConfigured()) {
+      const storageConfig = loadStorageConfig();
+      const client = createS3Client(storageConfig);
+      const key = exportObjectKey(row.userId, exportId);
+      await client.send(
+        new PutObjectCommand({
+          Bucket: storageConfig.S3_BUCKET,
+          Key: key,
+          Body: json,
+          ContentType: 'application/json',
+        }),
+      );
+      downloadUrl = `s3://${storageConfig.S3_BUCKET}/${key}`;
+    } else {
+      await redis.set(
+        `${EXPORT_BLOB_PREFIX}${exportId}`,
+        json,
+        'EX',
+        EXPORT_TTL_DAYS * 24 * 60 * 60,
+      );
+      const appConfig = getConfig();
+      downloadUrl = `${appConfig.API_URL}/account/compliance/export/${exportId}/download`;
+    }
+
+    await prisma.dataExportRequest.update({
+      where: { id: exportId },
+      data: {
+        status: 'READY',
+        downloadUrl,
+        expiresAt,
+        completedAt: new Date(),
+      },
+    });
+  } catch (error) {
+    logger.error({ exportId, error }, 'Data export failed');
+    await prisma.dataExportRequest.update({
+      where: { id: exportId },
+      data: { status: 'FAILED' },
+    });
+  }
+}
+
+async function executeAccountDeletion(requestId: string, lifecycleQueue: Queue): Promise<void> {
+  const request = await prisma.accountDeletionRequest.findUnique({
+    where: { id: requestId },
+  });
+  if (!request || request.status !== 'SCHEDULED') {
+    return;
+  }
+
+  const userId = request.userId;
+  const user = await prisma.user.findUnique({ where: { id: userId } });
+  if (!user || user.status === 'DELETED') {
+    await prisma.accountDeletionRequest.update({
+      where: { id: requestId },
+      data: { status: 'COMPLETED', completedAt: new Date() },
+    });
+    return;
+  }
+
+  const servers = await prisma.gameServer.findMany({
+    where: { userId, status: { not: 'DELETED' } },
+  });
+
+  for (const server of servers) {
+    if (server.status === 'RUNNING' || server.status === 'STARTING') {
+      continue;
+    }
+
+    if (ALLOWED_TRANSITIONS[server.status as GameServerStatus]?.includes('DELETING')) {
+      const correlationId = randomUUID();
+      await prisma.gameServer.update({
+        where: { id: server.id },
+        data: { status: 'DELETING' },
+      });
+      await lifecycleQueue.add(
+        'delete-server',
+        { serverId: server.id, action: 'delete', correlationId },
+        { jobId: correlationId },
+      );
+    }
+  }
+
+  const anonymizedId = randomUUID().slice(0, 8);
+  await prisma.$transaction([
+    prisma.userSession.updateMany({
+      where: { userId, revokedAt: null },
+      data: { revokedAt: new Date() },
+    }),
+    prisma.user.update({
+      where: { id: userId },
+      data: {
+        email: `deleted-${anonymizedId}@deleted.invalid`,
+        username: `deleted-${anonymizedId}`,
+        displayName: null,
+        status: 'DELETED',
+      },
+    }),
+    prisma.accountDeletionRequest.update({
+      where: { id: requestId },
+      data: { status: 'COMPLETED', completedAt: new Date() },
+    }),
+  ]);
+}
+
+async function collectUserExportData(userId: string) {
+  const [user, servers, auditEvents] = await Promise.all([
+    prisma.user.findUnique({
+      where: { id: userId },
+      select: {
+        id: true,
+        email: true,
+        username: true,
+        displayName: true,
+        status: true,
+        emailVerifiedAt: true,
+        createdAt: true,
+        updatedAt: true,
+      },
+    }),
+    prisma.gameServer.findMany({
+      where: { userId },
+      select: {
+        id: true,
+        name: true,
+        status: true,
+        edition: true,
+        softwareFamily: true,
+        minecraftVersion: true,
+        ramMb: true,
+        hostPort: true,
+        createdAt: true,
+        updatedAt: true,
+      },
+      orderBy: { createdAt: 'desc' },
+    }),
+    prisma.auditEvent.findMany({
+      where: { userId },
+      select: {
+        action: true,
+        entityType: true,
+        entityId: true,
+        createdAt: true,
+      },
+      orderBy: { createdAt: 'desc' },
+      take: 500,
+    }),
+  ]);
+
+  const actionCounts = auditEvents.reduce<Record<string, number>>((acc, event) => {
+    acc[event.action] = (acc[event.action] ?? 0) + 1;
+    return acc;
+  }, {});
+
+  return {
+    exportedAt: new Date().toISOString(),
+    profile: user,
+    servers,
+    auditSummary: {
+      totalEvents: auditEvents.length,
+      actionCounts,
+      recentEvents: auditEvents.slice(0, 50),
+    },
+  };
+}

apps/worker/src/handlers/server-lifecycle.ts

@@ -170,8 +170,31 @@ export async function processLifecycleJob(job: Job): Promise<{ status: string }>
       return processStartServerJob(job);
     case 'stop-server':
       return processStopServerJob(job);
+    case 'kill-server':
+      return processKillServerJob(job);
+    case 'delete-server':
+      return processDeleteServerJob(job);
     default:
       logger.warn({ jobName: job.name }, 'Unknown lifecycle job name');
       return { status: 'ignored' };
   }
 }
+
+async function processKillServerJob(job: Job): Promise<{ status: string }> {
+  const { serverId } = lifecycleJobSchema.parse(job.data);
+  await transitionServerStatus(serverId, 'STOPPED', {
+    reason: 'Kill completed',
+    correlationId: job.id ?? undefined,
+  });
+  return { status: 'killed' };
+}
+
+async function processDeleteServerJob(job: Job): Promise<{ status: string }> {
+  const { serverId } = lifecycleJobSchema.parse(job.data);
+  await transitionServerStatus(serverId, 'DELETED', {
+    expectedFrom: 'DELETING',
+    reason: 'Delete completed',
+    correlationId: job.id ?? undefined,
+  });
+  return { status: 'deleted' };
+}

apps/worker/src/index.ts

@@ -7,16 +7,24 @@ import { processDnsSyncTick } from './handlers/dns-sync';
 import { processIdleShutdownTick, processUsageMeteringTick } from './handlers/idle-billing';
 import { processNodeHealthTick } from './handlers/node-health';
 import { processNotificationJob } from './handlers/notifications';
+import { processNotificationDispatchJob } from './handlers/notification-dispatch';
 import { processAddonJob } from './handlers/server-addons';
 import { processBackupJob, processScheduledBackupsTick } from './handlers/server-backups';
 import { processLifecycleJob } from './handlers/server-lifecycle';
 import { processProvisionServerJob } from './handlers/server-provisioning';
 import { processStartQueueTick } from './handlers/start-queue';
 import { processWhmcsReconciliationTick, processWhmcsUsagePeriodTick } from './handlers/whmcs-sync';
+import {
+  processCatalogSyncTick,
+  processComplianceTick,
+  processScheduleTick,
+} from './handlers/schedules-compliance';
 import { startHealthServer } from './health';
 import { logger } from './logger';
 import { closeRedisConnections } from './redis';
 import {
+  QUEUE_CATALOG_SYNC,
+  QUEUE_DEAD_LETTER,
   QUEUE_NOTIFICATIONS,
   QUEUE_SERVER_ADDONS,
   QUEUE_SERVER_BACKUPS,
@@ -39,6 +47,9 @@ function createQueueWorker(
       );
 
       if (queueName === QUEUE_NOTIFICATIONS) {
+        if (job.name === 'dispatch-notification') {
+          return processNotificationDispatchJob(job);
+        }
         return processNotificationJob(job);
       }
 
@@ -58,6 +69,15 @@ function createQueueWorker(
         return processLifecycleJob(job);
       }
 
+      if (queueName === QUEUE_CATALOG_SYNC) {
+        return processCatalogSyncTick();
+      }
+
+      if (queueName === QUEUE_DEAD_LETTER) {
+        logger.warn({ jobId: job.id, data: job.data }, 'Dead-letter job recorded');
+        return { recorded: true };
+      }
+
       logger.warn({ queue: queueName, jobName: job.name }, 'Unhandled job');
     },
     { connection },
@@ -117,6 +137,15 @@ async function bootstrap(): Promise<void> {
     void processWhmcsUsagePeriodTick().catch((error: Error) => {
       logger.error({ err: error }, 'WHMCS usage period tick failed');
     });
+    void processScheduleTick().catch((error: Error) => {
+      logger.error({ err: error }, 'Schedule tick failed');
+    });
+    void processCatalogSyncTick().catch((error: Error) => {
+      logger.error({ err: error }, 'Catalog sync tick failed');
+    });
+    void processComplianceTick().catch((error: Error) => {
+      logger.error({ err: error }, 'Compliance tick failed');
+    });
   }, 60_000);
 
   const shutdown = async (signal: string): Promise<void> => {

apps/worker/src/queues.ts

@@ -3,6 +3,8 @@ export const QUEUE_SERVER_BACKUPS = 'server-backups' as const;
 export const QUEUE_SERVER_LIFECYCLE = 'server-lifecycle' as const;
 export const QUEUE_SERVER_PROVISIONING = 'server-provisioning' as const;
 export const QUEUE_NOTIFICATIONS = 'notifications' as const;
+export const QUEUE_CATALOG_SYNC = 'catalog-sync' as const;
+export const QUEUE_DEAD_LETTER = 'dead-letter' as const;
 
 import { Queue } from 'bullmq';
 
@@ -14,6 +16,8 @@ export const WORKER_QUEUES = [
   QUEUE_SERVER_LIFECYCLE,
   QUEUE_SERVER_PROVISIONING,
   QUEUE_NOTIFICATIONS,
+  QUEUE_CATALOG_SYNC,
+  QUEUE_DEAD_LETTER,
 ] as const;
 
 export type WorkerQueueName = (typeof WORKER_QUEUES)[number];
@@ -49,3 +53,19 @@ export function getBackupsQueue(): Queue {
 
   return backupsQueue;
 }
+
+let lifecycleQueue: Queue | null = null;
+
+export function getLifecycleQueue(): Queue {
+  if (!lifecycleQueue) {
+    const config = validateConfig();
+    lifecycleQueue = new Queue(QUEUE_SERVER_LIFECYCLE, {
+      connection: {
+        url: config.REDIS_URL,
+        maxRetriesPerRequest: null,
+      },
+    });
+  }
+
+  return lifecycleQueue;
+}