Phase6

apps/api/package.json

@@ -20,6 +20,7 @@
     "@hexahost/config": "workspace:*",
     "@hexahost/contracts": "workspace:*",
     "@hexahost/database": "workspace:*",
+    "@hexahost/scheduler": "workspace:*",
     "@hexahost/storage": "workspace:*",
     "@nestjs/common": "^11.0.7",
     "@nestjs/core": "^11.0.7",

apps/api/src/admin/admin-nodes.controller.ts

@@ -1,14 +1,19 @@
 import {
   Body,
   Controller,
+  Get,
   HttpCode,
   HttpStatus,
+  Param,
+  ParseUUIDPipe,
   Post,
   UseGuards,
 } from '@nestjs/common';
 import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger';
 import { z } from 'zod';
 
+import type { NodeListResponse, NodeSummary } from '@hexahost/contracts';
+
 import { CurrentUser } from '../auth/decorators/current-user.decorator';
 import { SessionGuard } from '../auth/guards/session.guard';
 import { ZodValidationPipe } from '../common/pipes/zod-validation.pipe';
@@ -20,6 +25,12 @@ const createNodeRequestSchema = z.object({
   name: z.string().min(1).max(64),
   hostname: z.string().min(1).max(255),
   maxServers: z.number().int().min(1).max(1000).optional(),
+  maxRamMb: z.number().int().min(2048).max(1048576).optional(),
+  platformRamMb: z.number().int().min(512).max(65536).optional(),
+});
+
+const maintenanceRequestSchema = z.object({
+  enabled: z.boolean(),
 });
 
 @ApiTags('admin')
@@ -28,6 +39,13 @@ const createNodeRequestSchema = z.object({
 export class AdminNodesController {
   constructor(private readonly adminNodesService: AdminNodesService) {}
 
+  @Get()
+  @ApiOperation({ summary: 'List game nodes (super admin only)' })
+  list(@CurrentUser() user: { roles: string[] }): Promise<NodeListResponse> {
+    this.adminNodesService.assertSuperAdmin(user.roles);
+    return this.adminNodesService.listNodes();
+  }
+
   @Post()
   @HttpCode(HttpStatus.CREATED)
   @ApiOperation({ summary: 'Register a new game node (super admin only)' })
@@ -42,4 +60,28 @@ export class AdminNodesController {
       body as Parameters<AdminNodesService['createNode']>[0],
     );
   }
+
+  @Post(':id/drain')
+  @HttpCode(HttpStatus.OK)
+  @ApiOperation({ summary: 'Drain a game node (super admin only)' })
+  drain(
+    @CurrentUser() user: { roles: string[] },
+    @Param('id', ParseUUIDPipe) id: string,
+  ): Promise<NodeSummary> {
+    this.adminNodesService.assertSuperAdmin(user.roles);
+    return this.adminNodesService.requestDrain(id);
+  }
+
+  @Post(':id/maintenance')
+  @HttpCode(HttpStatus.OK)
+  @ApiOperation({ summary: 'Toggle maintenance mode for a node (super admin only)' })
+  maintenance(
+    @CurrentUser() user: { roles: string[] },
+    @Param('id', ParseUUIDPipe) id: string,
+    @Body(new ZodValidationPipe(maintenanceRequestSchema)) body: unknown,
+  ): Promise<NodeSummary> {
+    this.adminNodesService.assertSuperAdmin(user.roles);
+    const input = body as z.infer<typeof maintenanceRequestSchema>;
+    return this.adminNodesService.setMaintenance(id, input.enabled);
+  }
 }

apps/api/src/admin/admin-nodes.service.ts

@@ -1,17 +1,27 @@
-import { ForbiddenException, Injectable } from '@nestjs/common';
+import {
+  ConflictException,
+  ForbiddenException,
+  Injectable,
+  NotFoundException,
+} from '@nestjs/common';
 
 import {
   generateSecureToken,
   hashToken,
   UserRole,
 } from '@hexahost/auth';
+import type { NodeListResponse, NodeSummary } from '@hexahost/contracts';
+import { computeAvailableRamMb } from '@hexahost/scheduler';
 
 import { PrismaService } from '../prisma/prisma.service';
+import { loadNodeSnapshots } from '../scheduling/node-snapshots';
 
 export interface CreateNodeInput {
   name: string;
   hostname: string;
   maxServers?: number;
+  maxRamMb?: number;
+  platformRamMb?: number;
 }
 
 export interface CreateNodeResponse {
@@ -21,6 +31,7 @@ export interface CreateNodeResponse {
     hostname: string;
     status: string;
     maxServers: number;
+    maxRamMb: number;
     createdAt: string;
   };
   enrollmentToken: string;
@@ -36,6 +47,19 @@ export class AdminNodesService {
     }
   }
 
+  async listNodes(): Promise<NodeListResponse> {
+    const snapshots = await loadNodeSnapshots();
+    const nodes = await this.prisma.gameNode.findMany({
+      orderBy: { createdAt: 'asc' },
+    });
+
+    const snapshotById = new Map(snapshots.map((snapshot) => [snapshot.id, snapshot]));
+
+    return {
+      nodes: nodes.map((node) => this.toSummary(node, snapshotById.get(node.id))),
+    };
+  }
+
   async createNode(input: CreateNodeInput): Promise<CreateNodeResponse> {
     const enrollmentToken = generateSecureToken();
 
@@ -44,6 +68,8 @@ export class AdminNodesService {
         name: input.name,
         hostname: input.hostname,
         maxServers: input.maxServers ?? 10,
+        maxRamMb: input.maxRamMb ?? 16384,
+        platformRamMb: input.platformRamMb ?? 2048,
         enrollmentTokenHash: hashToken(enrollmentToken),
       },
     });
@@ -55,9 +81,109 @@ export class AdminNodesService {
         hostname: node.hostname,
         status: node.status,
         maxServers: node.maxServers,
+        maxRamMb: node.maxRamMb,
         createdAt: node.createdAt.toISOString(),
       },
       enrollmentToken,
     };
   }
+
+  async requestDrain(nodeId: string): Promise<NodeSummary> {
+    const node = await this.requireNode(nodeId);
+
+    if (node.status === 'DRAINING' || node.status === 'MAINTENANCE') {
+      throw new ConflictException('Node is already draining or in maintenance');
+    }
+
+    const updated = await this.prisma.gameNode.update({
+      where: { id: nodeId },
+      data: {
+        status: 'DRAINING',
+        drainRequestedAt: new Date(),
+      },
+    });
+
+    const snapshot = (await loadNodeSnapshots([nodeId]))[0];
+    return this.toSummary(updated, snapshot);
+  }
+
+  async setMaintenance(nodeId: string, enabled: boolean): Promise<NodeSummary> {
+    const node = await this.requireNode(nodeId);
+
+    if (enabled && node.status === 'MAINTENANCE') {
+      const snapshot = (await loadNodeSnapshots([nodeId]))[0];
+      return this.toSummary(node, snapshot);
+    }
+
+    const updated = await this.prisma.gameNode.update({
+      where: { id: nodeId },
+      data: {
+        status: enabled ? 'MAINTENANCE' : 'OFFLINE',
+        drainRequestedAt: null,
+      },
+    });
+
+    const snapshot = (await loadNodeSnapshots([nodeId]))[0];
+    return this.toSummary(updated, snapshot);
+  }
+
+  private async requireNode(nodeId: string) {
+    const node = await this.prisma.gameNode.findUnique({ where: { id: nodeId } });
+    if (!node) {
+      throw new NotFoundException('Node not found');
+    }
+    return node;
+  }
+
+  private toSummary(
+    node: {
+      id: string;
+      name: string;
+      hostname: string;
+      status: NodeSummary['status'];
+      maxServers: number;
+      maxRamMb: number;
+      platformRamMb: number;
+      activeServers: number;
+      lastHeartbeatAt: Date | null;
+      agentVersion: string | null;
+      drainRequestedAt: Date | null;
+      createdAt: Date;
+    },
+    snapshot?: {
+      reservedRamMb: number;
+    },
+  ): NodeSummary {
+    const reservedRamMb = snapshot?.reservedRamMb ?? 0;
+    const availableRamMb = snapshot
+      ? computeAvailableRamMb({
+          id: node.id,
+          status: node.status,
+          maxServers: node.maxServers,
+          maxRamMb: node.maxRamMb,
+          platformRamMb: node.platformRamMb,
+          activeServers: node.activeServers,
+          lastHeartbeatAt: node.lastHeartbeatAt,
+          reservedRamMb,
+          activeAllocationCount: 0,
+        })
+      : Math.max(0, node.maxRamMb - node.platformRamMb - reservedRamMb);
+
+    return {
+      id: node.id,
+      name: node.name,
+      hostname: node.hostname,
+      status: node.status,
+      maxServers: node.maxServers,
+      maxRamMb: node.maxRamMb,
+      platformRamMb: node.platformRamMb,
+      activeServers: node.activeServers,
+      reservedRamMb,
+      availableRamMb,
+      lastHeartbeatAt: node.lastHeartbeatAt?.toISOString() ?? null,
+      agentVersion: node.agentVersion,
+      drainRequestedAt: node.drainRequestedAt?.toISOString() ?? null,
+      createdAt: node.createdAt.toISOString(),
+    };
+  }
 }

apps/api/src/nodes/nodes.service.ts

@@ -110,10 +110,18 @@ export class NodesService {
   }
 
   async markNodeOnline(nodeId: string, agentVersion?: string): Promise<void> {
+    const node = await this.prisma.gameNode.findUnique({ where: { id: nodeId } });
+    const nextStatus =
+      node?.status === 'UNREACHABLE' || node?.status === 'OFFLINE'
+        ? 'ONLINE'
+        : node?.status === 'MAINTENANCE' || node?.status === 'DRAINING'
+          ? node.status
+          : 'ONLINE';
+
     await this.prisma.gameNode.update({
       where: { id: nodeId },
       data: {
-        status: 'ONLINE',
+        status: nextStatus,
         lastHeartbeatAt: new Date(),
         enrolledAt: new Date(),
         agentVersion,
@@ -172,17 +180,35 @@ export class NodesService {
     payload?: Record<string, unknown>,
   ): Promise<void> {
     const now = new Date();
+    const node = await this.prisma.gameNode.findUnique({ where: { id: nodeId } });
+
+    const nextStatus =
+      node?.status === 'UNREACHABLE'
+        ? 'ONLINE'
+        : node?.status === 'MAINTENANCE' || node?.status === 'DRAINING'
+          ? node.status
+          : 'ONLINE';
 
     await this.prisma.$transaction([
       this.prisma.gameNode.update({
         where: { id: nodeId },
         data: {
           lastHeartbeatAt: now,
-          status: 'ONLINE',
+          status: nextStatus,
           agentVersion:
             typeof payload?.['agentVersion'] === 'string'
               ? payload['agentVersion']
               : undefined,
+          metadata: payload
+            ? ({
+                cpuUsagePercent: payload['cpuUsagePercent'],
+                memoryUsedBytes: payload['memoryUsedBytes'],
+                memoryTotalBytes: payload['memoryTotalBytes'],
+                diskUsedBytes: payload['diskUsedBytes'],
+                diskTotalBytes: payload['diskTotalBytes'],
+                runningServers: payload['runningServers'],
+              } as Prisma.InputJsonValue)
+            : undefined,
         },
       }),
       this.prisma.gameNodeHeartbeat.create({

apps/api/src/scheduling/node-snapshots.ts

@@ -0,0 +1,59 @@
+import { prisma } from '@hexahost/database';
+import type { NodeSnapshot } from '@hexahost/scheduler';
+
+function readMemoryField(
+  payload: Record<string, unknown> | null | undefined,
+  key: 'memoryTotalBytes' | 'memoryUsedBytes',
+): number | null {
+  const value = payload?.[key];
+  return typeof value === 'number' ? value : null;
+}
+
+export async function loadNodeSnapshots(
+  nodeIds?: string[],
+): Promise<NodeSnapshot[]> {
+  const nodes = await prisma.gameNode.findMany({
+    where: nodeIds ? { id: { in: nodeIds } } : undefined,
+    include: {
+      allocations: {
+        where: { status: 'ACTIVE' },
+        select: { ramMbReserved: true },
+      },
+      heartbeats: {
+        orderBy: { createdAt: 'desc' },
+        take: 1,
+        select: { payload: true },
+      },
+    },
+  });
+
+  return nodes.map((node) => {
+    const heartbeatPayload = node.heartbeats[0]?.payload as
+      | Record<string, unknown>
+      | undefined;
+
+    const reservedRamMb = node.allocations.reduce(
+      (sum, allocation) => sum + allocation.ramMbReserved,
+      0,
+    );
+
+    return {
+      id: node.id,
+      status: node.status,
+      maxServers: node.maxServers,
+      maxRamMb: node.maxRamMb,
+      platformRamMb: node.platformRamMb,
+      activeServers: node.activeServers,
+      lastHeartbeatAt: node.lastHeartbeatAt,
+      memoryTotalBytes: readMemoryField(heartbeatPayload, 'memoryTotalBytes'),
+      memoryUsedBytes: readMemoryField(heartbeatPayload, 'memoryUsedBytes'),
+      reservedRamMb,
+      activeAllocationCount: node.allocations.length,
+    };
+  });
+}
+
+export async function loadNodeSnapshot(nodeId: string): Promise<NodeSnapshot | null> {
+  const snapshots = await loadNodeSnapshots([nodeId]);
+  return snapshots[0] ?? null;
+}

apps/api/src/scheduling/scheduling.module.ts

@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common';
+
+import { PrismaModule } from '../prisma/prisma.module';
+
+import { SchedulingService } from './scheduling.service';
+
+@Module({
+  imports: [PrismaModule],
+  providers: [SchedulingService],
+  exports: [SchedulingService],
+})
+export class SchedulingModule {}

apps/api/src/scheduling/scheduling.service.ts

@@ -0,0 +1,175 @@
+import { randomUUID } from 'node:crypto';
+
+import { Injectable } from '@nestjs/common';
+
+import type { StartQueueResponse } from '@hexahost/contracts';
+import type { GameServer } from '@hexahost/database';
+import { canStartOnNode } from '@hexahost/scheduler';
+
+import { PrismaService } from '../prisma/prisma.service';
+
+import { loadNodeSnapshot } from './node-snapshots';
+
+const START_QUEUE_EXPIRY_MS = 30 * 60 * 1000;
+
+@Injectable()
+export class SchedulingService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  async canStartServer(server: GameServer): Promise<boolean> {
+    if (!server.nodeId) {
+      return false;
+    }
+
+    const snapshot = await loadNodeSnapshot(server.nodeId);
+    return snapshot ? canStartOnNode(snapshot, server.ramMb) : false;
+  }
+
+  async enqueueStart(
+    server: GameServer,
+    actorId: string,
+    correlationId: string,
+  ): Promise<{ server: GameServer; queuePosition: number }> {
+    const expiresAt = new Date(Date.now() + START_QUEUE_EXPIRY_MS);
+
+    const updated = await this.prisma.$transaction(async (tx) => {
+      await tx.serverStartQueueEntry.upsert({
+        where: { serverId: server.id },
+        create: {
+          serverId: server.id,
+          nodeId: server.nodeId!,
+          status: 'QUEUED',
+          correlationId,
+          expiresAt,
+        },
+        update: {
+          status: 'QUEUED',
+          correlationId,
+          expiresAt,
+          requestedAt: new Date(),
+        },
+      });
+
+      const next = await tx.gameServer.update({
+        where: { id: server.id },
+        data: {
+          status: 'QUEUED',
+          version: { increment: 1 },
+        },
+      });
+
+      await tx.gameServerStateTransition.create({
+        data: {
+          gameServerId: server.id,
+          fromStatus: server.status,
+          toStatus: 'QUEUED',
+          reason: 'Start queued due to insufficient node capacity',
+          actorId,
+          correlationId,
+        },
+      });
+
+      return next;
+    });
+
+    const queuePosition = await this.getQueuePosition(server.id);
+
+    return { server: updated, queuePosition };
+  }
+
+  async getQueuePosition(serverId: string): Promise<number> {
+    const entry = await this.prisma.serverStartQueueEntry.findUnique({
+      where: { serverId },
+    });
+
+    if (!entry || entry.status !== 'QUEUED') {
+      return 0;
+    }
+
+    const ahead = await this.prisma.serverStartQueueEntry.count({
+      where: {
+        status: 'QUEUED',
+        OR: [
+          { priority: { gt: entry.priority } },
+          {
+            priority: entry.priority,
+            requestedAt: { lt: entry.requestedAt },
+          },
+        ],
+      },
+    });
+
+    return ahead + 1;
+  }
+
+  async getStartQueue(userId: string, serverId: string): Promise<StartQueueResponse> {
+    const server = await this.prisma.gameServer.findFirst({
+      where: { id: serverId, userId },
+    });
+
+    if (!server) {
+      return { entry: null };
+    }
+
+    const entry = await this.prisma.serverStartQueueEntry.findUnique({
+      where: { serverId },
+    });
+
+    if (!entry || entry.status !== 'QUEUED') {
+      return { entry: null };
+    }
+
+    const position = await this.getQueuePosition(serverId);
+
+    return {
+      entry: {
+        serverId: entry.serverId,
+        nodeId: entry.nodeId,
+        status: entry.status,
+        priority: entry.priority,
+        position,
+        requestedAt: entry.requestedAt.toISOString(),
+        expiresAt: entry.expiresAt?.toISOString() ?? null,
+      },
+    };
+  }
+
+  async cancelQueuedStart(userId: string, serverId: string): Promise<GameServer> {
+    const server = await this.prisma.gameServer.findFirst({
+      where: { id: serverId, userId },
+    });
+
+    if (!server) {
+      throw new Error('Server not found');
+    }
+
+    if (server.status !== 'QUEUED') {
+      return server;
+    }
+
+    return this.prisma.$transaction(async (tx) => {
+      await tx.serverStartQueueEntry.updateMany({
+        where: { serverId, status: 'QUEUED' },
+        data: { status: 'CANCELLED' },
+      });
+
+      const updated = await tx.gameServer.update({
+        where: { id: serverId },
+        data: { status: 'STOPPED', version: { increment: 1 } },
+      });
+
+      await tx.gameServerStateTransition.create({
+        data: {
+          gameServerId: serverId,
+          fromStatus: 'QUEUED',
+          toStatus: 'STOPPED',
+          reason: 'Queued start cancelled by user',
+          actorId: userId,
+          correlationId: randomUUID(),
+        },
+      });
+
+      return updated;
+    });
+  }
+}

apps/api/src/servers/server-state.service.ts

@@ -7,13 +7,15 @@ const ALLOWED_TRANSITIONS: Record<GameServerStatus, GameServerStatus[]> = {
   DRAFT: ['PROVISIONING', 'DELETING'],
   PROVISIONING: ['INSTALLING', 'ERROR', 'DELETING'],
   INSTALLING: ['STOPPED', 'ERROR', 'DELETING'],
-  STOPPED: ['STARTING', 'PROVISIONING', 'BACKING_UP', 'RESTORING', 'DELETING'],
-  STARTING: ['RUNNING', 'STOPPED', 'ERROR', 'DELETING'],
-  RUNNING: ['STOPPING', 'BACKING_UP', 'ERROR', 'DELETING'],
-  STOPPING: ['STOPPED', 'ERROR', 'DELETING'],
+  STOPPED: ['STARTING', 'QUEUED', 'PROVISIONING', 'BACKING_UP', 'RESTORING', 'DELETING'],
+  QUEUED: ['STARTING', 'STOPPED', 'DELETING'],
+  STARTING: ['RUNNING', 'STOPPED', 'ERROR', 'UNKNOWN', 'DELETING'],
+  RUNNING: ['STOPPING', 'BACKING_UP', 'ERROR', 'UNKNOWN', 'DELETING'],
+  STOPPING: ['STOPPED', 'ERROR', 'UNKNOWN', 'DELETING'],
   BACKING_UP: ['RUNNING', 'STOPPED', 'ERROR'],
   RESTORING: ['STOPPED', 'ERROR'],
-  ERROR: ['STOPPED', 'STARTING', 'PROVISIONING', 'DELETING'],
+  UNKNOWN: ['STOPPED', 'STARTING', 'ERROR', 'DELETING'],
+  ERROR: ['STOPPED', 'STARTING', 'QUEUED', 'PROVISIONING', 'DELETING'],
   DELETING: ['DELETED'],
   DELETED: [],
 };
@@ -55,6 +57,8 @@ export class ServerStateService {
       case 'ERROR':
         return 'PROVISIONING';
       case 'STOPPED':
+      case 'UNKNOWN':
+      case 'QUEUED':
         return 'STARTING';
       default:
         throw new BadRequestException(

apps/api/src/servers/servers.controller.ts

@@ -85,4 +85,23 @@ export class ServersController {
   ) {
     return this.serversService.restartServer(user.id, id, skip === 'true');
   }
+
+  @Get(':id/start-queue')
+  @ApiOperation({ summary: 'Get start queue status for a server' })
+  startQueue(
+    @CurrentUser() user: { id: string },
+    @Param('id', ParseUUIDPipe) id: string,
+  ) {
+    return this.serversService.getStartQueue(user.id, id);
+  }
+
+  @Post(':id/cancel-queue')
+  @HttpCode(HttpStatus.OK)
+  @ApiOperation({ summary: 'Cancel a queued server start' })
+  cancelQueue(
+    @CurrentUser() user: { id: string },
+    @Param('id', ParseUUIDPipe) id: string,
+  ) {
+    return this.serversService.cancelQueuedStart(user.id, id);
+  }
 }

apps/api/src/servers/servers.module.ts

@@ -8,6 +8,8 @@ import { AuthModule } from '../auth/auth.module';
 import { NodeBridgeModule } from '../nodes/node-bridge.module';
 import { AppConfigModule } from '../config/app-config.module';
 
+import { SchedulingModule } from '../scheduling/scheduling.module';
+
 import { ConsoleModule } from './console/console.module';
 import { AddonsModule } from './addons/addons.module';
 import { BackupsModule } from './backups/backups.module';
@@ -30,6 +32,7 @@ import { PlansService } from './plans.service';
   imports: [
     AppConfigModule,
     AuthModule,
+    SchedulingModule,
     forwardRef(() => ConsoleModule),
     forwardRef(() => AddonsModule),
     forwardRef(() => BackupsModule),

apps/api/src/servers/servers.service.ts

@@ -19,6 +19,7 @@ import type { GameServer } from '@hexahost/database';
 type GameServerStatus = GameServer['status'];
 
 import { PrismaService } from '../prisma/prisma.service';
+import { SchedulingService } from '../scheduling/scheduling.service';
 
 import { findOwnedServer } from './shared/server-ownership.util';
 import { ServerStateService } from './server-state.service';
@@ -34,6 +35,7 @@ export class ServersService {
   constructor(
     private readonly prisma: PrismaService,
     private readonly serverState: ServerStateService,
+    private readonly scheduling: SchedulingService,
     @Inject(REDIS_CLIENT) private readonly redis: Redis,
     @Inject(SERVER_LIFECYCLE_QUEUE)
     private readonly lifecycleQueue: Queue,
@@ -106,6 +108,22 @@ export class ServersService {
     this.serverState.assertTransition(server.status, targetStatus);
 
     const correlationId = randomUUID();
+
+    if (targetStatus === 'STARTING') {
+      const canStart = await this.scheduling.canStartServer(server);
+      if (!canStart) {
+        const queued = await this.scheduling.enqueueStart(
+          server,
+          userId,
+          correlationId,
+        );
+        return {
+          server: this.toResponse(queued.server),
+          queuePosition: queued.queuePosition,
+        };
+      }
+    }
+
     const updated = await this.transitionServer(
       server,
       targetStatus,
@@ -153,6 +171,19 @@ export class ServersService {
     };
   }
 
+  async getStartQueue(userId: string, serverId: string) {
+    return this.scheduling.getStartQueue(userId, serverId);
+  }
+
+  async cancelQueuedStart(
+    userId: string,
+    serverId: string,
+  ): Promise<ServerActionResponse> {
+    await findOwnedServer(this.prisma, userId, serverId);
+    const updated = await this.scheduling.cancelQueuedStart(userId, serverId);
+    return { server: this.toResponse(updated) };
+  }
+
   async stopServer(
     userId: string,
     serverId: string,

apps/api/test/phase6.test.js

@@ -0,0 +1,41 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+describe('phase 6 api contract', () => {
+  it('documents scheduler, queue and admin node routes', () => {
+    const files = [
+      'scheduling/scheduling.service.ts',
+      'admin/admin-nodes.controller.ts',
+      'admin/admin-nodes.service.ts',
+    ];
+
+    for (const file of files) {
+      const source = readFileSync(join(process.cwd(), 'src', file), 'utf8');
+      assert.ok(source.length > 0, `${file} should exist`);
+    }
+
+    const adminSource = readFileSync(
+      join(process.cwd(), 'src', 'admin/admin-nodes.controller.ts'),
+      'utf8',
+    );
+    assert.ok(adminSource.includes("@Get()"));
+    assert.ok(adminSource.includes(':id/drain'));
+    assert.ok(adminSource.includes(':id/maintenance'));
+
+    const schedulingSource = readFileSync(
+      join(process.cwd(), 'src', 'scheduling/scheduling.service.ts'),
+      'utf8',
+    );
+    assert.ok(schedulingSource.includes('enqueueStart'));
+    assert.ok(schedulingSource.includes('getQueuePosition'));
+
+    const serversSource = readFileSync(
+      join(process.cwd(), 'src', 'servers/servers.controller.ts'),
+      'utf8',
+    );
+    assert.ok(serversSource.includes('start-queue'));
+    assert.ok(serversSource.includes('cancel-queue'));
+  });
+});

apps/web/messages/de.json

@@ -163,9 +163,11 @@
       "PROVISIONING": "Provisionierung",
       "INSTALLING": "Installation",
       "STOPPED": "Gestoppt",
+      "QUEUED": "In Warteschlange",
       "STARTING": "Startet",
       "RUNNING": "Läuft",
       "STOPPING": "Stoppt",
+      "UNKNOWN": "Unbekannt",
       "ERROR": "Fehler",
       "DELETING": "Wird gelöscht",
       "DELETED": "Gelöscht"

apps/web/messages/en.json

@@ -163,9 +163,11 @@
       "PROVISIONING": "Provisioning",
       "INSTALLING": "Installing",
       "STOPPED": "Stopped",
+      "QUEUED": "Queued",
       "STARTING": "Starting",
       "RUNNING": "Running",
       "STOPPING": "Stopping",
+      "UNKNOWN": "Unknown",
       "ERROR": "Error",
       "DELETING": "Deleting",
       "DELETED": "Deleted"

apps/worker/package.json

@@ -15,6 +15,7 @@
     "@hexahost/catalog": "workspace:*",
     "@hexahost/config": "workspace:*",
     "@hexahost/database": "workspace:*",
+    "@hexahost/scheduler": "workspace:*",
     "@hexahost/storage": "workspace:*",
     "bullmq": "^5.34.10",
     "ioredis": "^5.4.2",

apps/worker/src/allocation.ts

@@ -0,0 +1,22 @@
+import { prisma } from '@hexahost/database';
+
+export async function releaseServerAllocation(serverId: string): Promise<void> {
+  const allocation = await prisma.gameServerAllocation.findUnique({
+    where: { serverId },
+  });
+
+  if (!allocation || allocation.status === 'RELEASED') {
+    return;
+  }
+
+  await prisma.$transaction([
+    prisma.gameServerAllocation.update({
+      where: { serverId },
+      data: { status: 'RELEASED' },
+    }),
+    prisma.gameNode.updateMany({
+      where: { id: allocation.nodeId, activeServers: { gt: 0 } },
+      data: { activeServers: { decrement: 1 } },
+    }),
+  ]);
+}

apps/worker/src/handlers/node-health.ts

@@ -0,0 +1,105 @@
+import { prisma } from '@hexahost/database';
+import { HEARTBEAT_STALE_MS } from '@hexahost/scheduler';
+
+import { logger } from '../logger';
+import { transitionServerStatus } from '../server-state';
+
+const ACTIVE_SERVER_STATUSES = [
+  'RUNNING',
+  'STARTING',
+  'STOPPING',
+] as const;
+
+export async function processNodeHealthTick(): Promise<void> {
+  const cutoff = new Date(Date.now() - HEARTBEAT_STALE_MS);
+
+  const staleNodes = await prisma.gameNode.findMany({
+    where: {
+      status: { in: ['ONLINE', 'DRAINING', 'MAINTENANCE'] },
+      OR: [{ lastHeartbeatAt: null }, { lastHeartbeatAt: { lt: cutoff } }],
+    },
+    select: { id: true, name: true },
+  });
+
+  for (const node of staleNodes) {
+    await markNodeUnreachable(node.id);
+    logger.warn({ nodeId: node.id, name: node.name }, 'Node marked unreachable');
+  }
+
+  const drainingNodes = await prisma.gameNode.findMany({
+    where: { status: 'DRAINING' },
+    include: {
+      allocations: {
+        where: { status: 'ACTIVE' },
+        select: { id: true },
+      },
+    },
+  });
+
+  for (const node of drainingNodes) {
+    if (node.allocations.length === 0 && node.drainRequestedAt) {
+      await prisma.gameNode.update({
+        where: { id: node.id },
+        data: {
+          status: 'MAINTENANCE',
+          drainRequestedAt: null,
+        },
+      });
+
+      logger.info({ nodeId: node.id }, 'Drain complete; node set to maintenance');
+    }
+  }
+}
+
+async function markNodeUnreachable(nodeId: string): Promise<void> {
+  await prisma.$transaction(async (tx) => {
+    const node = await tx.gameNode.findUnique({ where: { id: nodeId } });
+    if (!node || node.status === 'UNREACHABLE') {
+      return;
+    }
+
+    await tx.gameNode.update({
+      where: { id: nodeId },
+      data: { status: 'UNREACHABLE' },
+    });
+
+    const affectedServers = await tx.gameServer.findMany({
+      where: {
+        nodeId,
+        status: { in: [...ACTIVE_SERVER_STATUSES] },
+      },
+      select: { id: true, status: true },
+    });
+
+    for (const server of affectedServers) {
+      await tx.gameServer.update({
+        where: { id: server.id },
+        data: { status: 'UNKNOWN', version: { increment: 1 } },
+      });
+
+      await tx.gameServerStateTransition.create({
+        data: {
+          gameServerId: server.id,
+          fromStatus: server.status,
+          toStatus: 'UNKNOWN',
+          reason: 'Node heartbeat lost',
+          errorCode: 'NODE_UNREACHABLE',
+        },
+      });
+    }
+  });
+}
+
+export async function reconcileNodeOnline(nodeId: string): Promise<void> {
+  const node = await prisma.gameNode.findUnique({ where: { id: nodeId } });
+  if (!node || node.status !== 'UNREACHABLE') {
+    return;
+  }
+
+  await prisma.gameNode.update({
+    where: { id: nodeId },
+    data: { status: 'ONLINE' },
+  });
+
+  logger.info({ nodeId }, 'Node reconciled back to online');
+}

apps/worker/src/handlers/server-lifecycle.ts

@@ -1,11 +1,14 @@
 import { randomUUID } from 'node:crypto';
 
+import { canStartOnNode } from '@hexahost/scheduler';
 import { Job } from 'bullmq';
 import { z } from 'zod';
 
 import { prisma } from '@hexahost/database';
 
+import { defaultQueueExpiry } from './start-queue';
 import { logger } from '../logger';
+import { loadNodeSnapshot } from '../node-snapshots';
 import { publishNodeCommand } from '../redis';
 import { transitionServerStatus } from '../server-state';
 
@@ -64,10 +67,56 @@ export async function processStartServerJob(job: Job): Promise<{ status: string
   const { serverId } = lifecycleJobSchema.parse(job.data);
   const server = await loadServerForLifecycle(serverId);
 
-  if (server.status !== 'STOPPED') {
+  if (server.status !== 'STOPPED' && server.status !== 'QUEUED') {
     throw new Error(`Cannot start server ${serverId} from status ${server.status}`);
   }
 
+  const requiredRamMb = server.ramMb;
+  const nodeSnapshot = await loadNodeSnapshot(server.nodeId!);
+
+  if (!nodeSnapshot || !canStartOnNode(nodeSnapshot, requiredRamMb)) {
+    const correlationId = randomUUID();
+
+    await prisma.$transaction(async (tx) => {
+      await tx.serverStartQueueEntry.upsert({
+        where: { serverId },
+        create: {
+          serverId,
+          nodeId: server.nodeId!,
+          status: 'QUEUED',
+          correlationId,
+          expiresAt: defaultQueueExpiry(),
+        },
+        update: {
+          status: 'QUEUED',
+          correlationId,
+          expiresAt: defaultQueueExpiry(),
+          requestedAt: new Date(),
+        },
+      });
+
+      if (server.status !== 'QUEUED') {
+        await tx.gameServer.update({
+          where: { id: serverId },
+          data: { status: 'QUEUED', version: { increment: 1 } },
+        });
+
+        await tx.gameServerStateTransition.create({
+          data: {
+            gameServerId: serverId,
+            fromStatus: server.status,
+            toStatus: 'QUEUED',
+            reason: 'Insufficient node capacity; queued for start',
+            correlationId,
+          },
+        });
+      }
+    });
+
+    logger.info({ serverId, nodeId: server.nodeId }, 'Server start queued');
+    return { status: 'queued' };
+  }
+
   const messageId = await publishLifecycleCommand(
     serverId,
     server.nodeId!,
@@ -76,7 +125,7 @@ export async function processStartServerJob(job: Job): Promise<{ status: string
   );
 
   await transitionServerStatus(serverId, 'STARTING', {
-    expectedFrom: 'STOPPED',
+    expectedFrom: ['STOPPED', 'QUEUED'],
     reason: 'Start requested',
     correlationId: messageId,
   });

apps/worker/src/handlers/server-provisioning.ts

@@ -1,10 +1,13 @@
 import { randomUUID } from 'node:crypto';
 
 import { prisma } from '@hexahost/database';
+import { selectNodeForAllocation } from '@hexahost/scheduler';
 import { Job } from 'bullmq';
 import { z } from 'zod';
 
+import { releaseServerAllocation } from '../allocation';
 import { logger } from '../logger';
+import { loadNodeSnapshots } from '../node-snapshots';
 import { publishNodeCommand, waitForNodeResponse } from '../redis';
 import { transitionServerStatus } from '../server-state';
 
@@ -65,10 +68,9 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
     throw new Error(`Server not found: ${serverId}`);
   }
 
-  const node = await prisma.gameNode.findFirst({
-    where: { status: 'ONLINE' },
-    orderBy: { createdAt: 'asc' },
-  });
+  const nodeSnapshots = await loadNodeSnapshots();
+  const ramMb = server.plan?.maxRamMb ?? server.ramMb;
+  const node = selectNodeForAllocation(nodeSnapshots, ramMb);
 
   if (!node) {
     await transitionServerStatus(serverId, 'ERROR', {
@@ -80,7 +82,6 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
   }
 
   const hostPort = await findAvailableHostPort(node.id);
-  const ramMb = server.plan?.maxRamMb ?? server.ramMb;
   const dataPath = buildDataPath(serverId);
   const messageId = randomUUID();
 
@@ -95,6 +96,11 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
       },
     });
 
+    await tx.gameNode.update({
+      where: { id: node.id },
+      data: { activeServers: { increment: 1 } },
+    });
+
     await tx.gameServer.update({
       where: { id: serverId },
       data: {
@@ -150,6 +156,7 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
   }
 
   if (!response.success) {
+    await releaseServerAllocation(serverId);
     await transitionServerStatus(serverId, 'ERROR', {
       reason: 'Provision failed',
       correlationId: messageId,

apps/worker/src/handlers/start-queue.ts

@@ -0,0 +1,126 @@
+import { randomUUID } from 'node:crypto';
+
+import { prisma } from '@hexahost/database';
+import { canStartOnNode } from '@hexahost/scheduler';
+import { Queue } from 'bullmq';
+
+import { logger } from '../logger';
+import { loadNodeSnapshot } from '../node-snapshots';
+import { transitionServerStatus } from '../server-state';
+import { QUEUE_SERVER_LIFECYCLE } from '../queues';
+import { getRedisPublisher } from '../redis';
+
+const START_QUEUE_EXPIRY_MS = 30 * 60 * 1000;
+
+let lifecycleQueue: Queue | null = null;
+
+function getLifecycleQueue(): Queue {
+  if (!lifecycleQueue) {
+    lifecycleQueue = new Queue(QUEUE_SERVER_LIFECYCLE, {
+      connection: getRedisPublisher(),
+    });
+  }
+
+  return lifecycleQueue;
+}
+
+export async function processStartQueueTick(): Promise<void> {
+  const entries = await prisma.serverStartQueueEntry.findMany({
+    where: { status: 'QUEUED' },
+    orderBy: [{ priority: 'desc' }, { requestedAt: 'asc' }],
+    include: {
+      server: {
+        include: { plan: true },
+      },
+    },
+    take: 25,
+  });
+
+  if (entries.length === 0) {
+    return;
+  }
+
+  const now = Date.now();
+
+  for (const entry of entries) {
+    if (entry.expiresAt && entry.expiresAt.getTime() < now) {
+      await expireQueueEntry(entry.serverId);
+      continue;
+    }
+
+    const requiredRamMb = entry.server.plan?.maxRamMb ?? entry.server.ramMb;
+    const nodeSnapshot = await loadNodeSnapshot(entry.nodeId);
+
+    if (!nodeSnapshot || !canStartOnNode(nodeSnapshot, requiredRamMb)) {
+      continue;
+    }
+
+    const correlationId = entry.correlationId ?? randomUUID();
+
+    await prisma.$transaction(async (tx) => {
+      const current = await tx.serverStartQueueEntry.findUnique({
+        where: { id: entry.id },
+      });
+
+      if (!current || current.status !== 'QUEUED') {
+        return;
+      }
+
+      await tx.serverStartQueueEntry.update({
+        where: { id: entry.id },
+        data: { status: 'DISPATCHED', correlationId },
+      });
+    });
+
+    await getLifecycleQueue().add(
+      'start-server',
+      { serverId: entry.serverId, correlationId },
+      { jobId: correlationId },
+    );
+
+    logger.info(
+      { serverId: entry.serverId, nodeId: entry.nodeId },
+      'Dispatched queued server start',
+    );
+  }
+}
+
+async function expireQueueEntry(serverId: string): Promise<void> {
+  await prisma.$transaction(async (tx) => {
+    const entry = await tx.serverStartQueueEntry.findUnique({
+      where: { serverId },
+    });
+
+    if (!entry || entry.status !== 'QUEUED') {
+      return;
+    }
+
+    await tx.serverStartQueueEntry.update({
+      where: { serverId },
+      data: { status: 'EXPIRED' },
+    });
+
+    const server = await tx.gameServer.findUnique({ where: { id: serverId } });
+    if (server?.status === 'QUEUED') {
+      await tx.gameServer.update({
+        where: { id: serverId },
+        data: { status: 'STOPPED', version: { increment: 1 } },
+      });
+
+      await tx.gameServerStateTransition.create({
+        data: {
+          gameServerId: serverId,
+          fromStatus: 'QUEUED',
+          toStatus: 'STOPPED',
+          reason: 'Start queue entry expired',
+        },
+      });
+    }
+  });
+
+  logger.info({ serverId }, 'Expired queued start request');
+}
+
+export function defaultQueueExpiry(): Date {
+  return new Date(Date.now() + START_QUEUE_EXPIRY_MS);
+}

apps/worker/src/index.ts

@@ -3,11 +3,13 @@ import { Job, Worker, type ConnectionOptions } from 'bullmq';
 import { validateConfig } from '@hexahost/config';
 import { prisma } from '@hexahost/database';
 
+import { processNodeHealthTick } from './handlers/node-health';
 import { processNotificationJob } from './handlers/notifications';
 import { processAddonJob } from './handlers/server-addons';
 import { processBackupJob, processScheduledBackupsTick } from './handlers/server-backups';
 import { processLifecycleJob } from './handlers/server-lifecycle';
 import { processProvisionServerJob } from './handlers/server-provisioning';
+import { processStartQueueTick } from './handlers/start-queue';
 import { startHealthServer } from './health';
 import { logger } from './logger';
 import { closeRedisConnections } from './redis';
@@ -91,6 +93,12 @@ async function bootstrap(): Promise<void> {
     void processScheduledBackupsTick().catch((error: Error) => {
       logger.error({ err: error }, 'Scheduled backup tick failed');
     });
+    void processStartQueueTick().catch((error: Error) => {
+      logger.error({ err: error }, 'Start queue tick failed');
+    });
+    void processNodeHealthTick().catch((error: Error) => {
+      logger.error({ err: error }, 'Node health tick failed');
+    });
   }, 60_000);
 
   const shutdown = async (signal: string): Promise<void> => {

apps/worker/src/node-snapshots.ts

@@ -0,0 +1,59 @@
+import { prisma } from '@hexahost/database';
+import type { NodeSnapshot } from '@hexahost/scheduler';
+
+function readMemoryField(
+  payload: Record<string, unknown> | null | undefined,
+  key: 'memoryTotalBytes' | 'memoryUsedBytes',
+): number | null {
+  const value = payload?.[key];
+  return typeof value === 'number' ? value : null;
+}
+
+export async function loadNodeSnapshots(
+  nodeIds?: string[],
+): Promise<NodeSnapshot[]> {
+  const nodes = await prisma.gameNode.findMany({
+    where: nodeIds ? { id: { in: nodeIds } } : undefined,
+    include: {
+      allocations: {
+        where: { status: 'ACTIVE' },
+        select: { ramMbReserved: true },
+      },
+      heartbeats: {
+        orderBy: { createdAt: 'desc' },
+        take: 1,
+        select: { payload: true },
+      },
+    },
+  });
+
+  return nodes.map((node) => {
+    const heartbeatPayload = node.heartbeats[0]?.payload as
+      | Record<string, unknown>
+      | undefined;
+
+    const reservedRamMb = node.allocations.reduce(
+      (sum, allocation) => sum + allocation.ramMbReserved,
+      0,
+    );
+
+    return {
+      id: node.id,
+      status: node.status,
+      maxServers: node.maxServers,
+      maxRamMb: node.maxRamMb,
+      platformRamMb: node.platformRamMb,
+      activeServers: node.activeServers,
+      lastHeartbeatAt: node.lastHeartbeatAt,
+      memoryTotalBytes: readMemoryField(heartbeatPayload, 'memoryTotalBytes'),
+      memoryUsedBytes: readMemoryField(heartbeatPayload, 'memoryUsedBytes'),
+      reservedRamMb,
+      activeAllocationCount: node.allocations.length,
+    };
+  });
+}
+
+export async function loadNodeSnapshot(nodeId: string): Promise<NodeSnapshot | null> {
+  const snapshots = await loadNodeSnapshots([nodeId]);
+  return snapshots[0] ?? null;
+}