Phase6

packages/contracts/src/index.ts

@@ -180,3 +180,17 @@ export {
   type InstallAddonRequest,
   type InstallAddonResponse,
 } from './addons';
+
+export {
+  gameNodeStatusSchema,
+  nodeSummarySchema,
+  nodeListResponseSchema,
+  startQueueStatusSchema,
+  startQueueEntrySchema,
+  startQueueResponseSchema,
+  type GameNodeStatus,
+  type NodeSummary,
+  type NodeListResponse,
+  type StartQueueEntry,
+  type StartQueueResponse,
+} from './nodes';

packages/contracts/src/nodes.ts

@@ -0,0 +1,57 @@
+import { z } from 'zod';
+
+export const gameNodeStatusSchema = z.enum([
+  'ONLINE',
+  'OFFLINE',
+  'MAINTENANCE',
+  'DRAINING',
+  'UNREACHABLE',
+]);
+
+export const nodeSummarySchema = z.object({
+  id: z.string().uuid(),
+  name: z.string(),
+  hostname: z.string(),
+  status: gameNodeStatusSchema,
+  maxServers: z.number().int(),
+  maxRamMb: z.number().int(),
+  platformRamMb: z.number().int(),
+  activeServers: z.number().int(),
+  reservedRamMb: z.number().int(),
+  availableRamMb: z.number().int(),
+  lastHeartbeatAt: z.string().datetime().nullable(),
+  agentVersion: z.string().nullable(),
+  drainRequestedAt: z.string().datetime().nullable(),
+  createdAt: z.string().datetime(),
+});
+
+export const nodeListResponseSchema = z.object({
+  nodes: z.array(nodeSummarySchema),
+});
+
+export const startQueueStatusSchema = z.enum([
+  'QUEUED',
+  'DISPATCHED',
+  'CANCELLED',
+  'EXPIRED',
+]);
+
+export const startQueueEntrySchema = z.object({
+  serverId: z.string().uuid(),
+  nodeId: z.string().uuid(),
+  status: startQueueStatusSchema,
+  priority: z.number().int(),
+  position: z.number().int().nullable(),
+  requestedAt: z.string().datetime(),
+  expiresAt: z.string().datetime().nullable(),
+});
+
+export const startQueueResponseSchema = z.object({
+  entry: startQueueEntrySchema.nullable(),
+});
+
+export type GameNodeStatus = z.infer<typeof gameNodeStatusSchema>;
+export type NodeSummary = z.infer<typeof nodeSummarySchema>;
+export type NodeListResponse = z.infer<typeof nodeListResponseSchema>;
+export type StartQueueEntry = z.infer<typeof startQueueEntrySchema>;
+export type StartQueueResponse = z.infer<typeof startQueueResponseSchema>;

packages/contracts/src/servers.ts

@@ -5,11 +5,13 @@ export const gameServerStatusSchema = z.enum([
   'PROVISIONING',
   'INSTALLING',
   'STOPPED',
+  'QUEUED',
   'STARTING',
   'RUNNING',
   'STOPPING',
   'BACKING_UP',
   'RESTORING',
+  'UNKNOWN',
   'ERROR',
   'DELETING',
   'DELETED',
@@ -68,6 +70,7 @@ export const serverListResponseSchema = z.object({
 export const serverActionResponseSchema = z.object({
   server: serverResponseSchema,
   jobId: z.string().optional(),
+  queuePosition: z.number().int().optional(),
 });
 
 export type GameServerStatus = z.infer<typeof gameServerStatusSchema>;

packages/database/prisma/migrations/20260626200000_phase6_multinode/migration.sql

@@ -0,0 +1,38 @@
+-- Phase 6: Multi-node scheduling, start queue, node capacity
+
+CREATE TYPE "StartQueueStatus" AS ENUM ('QUEUED', 'DISPATCHED', 'CANCELLED', 'EXPIRED');
+
+ALTER TYPE "GameServerStatus" ADD VALUE IF NOT EXISTS 'QUEUED';
+ALTER TYPE "GameServerStatus" ADD VALUE IF NOT EXISTS 'UNKNOWN';
+
+ALTER TABLE "game_nodes"
+  ADD COLUMN IF NOT EXISTS "maxRamMb" INTEGER NOT NULL DEFAULT 16384,
+  ADD COLUMN IF NOT EXISTS "platformRamMb" INTEGER NOT NULL DEFAULT 2048,
+  ADD COLUMN IF NOT EXISTS "drainRequestedAt" TIMESTAMPTZ(3);
+
+CREATE TABLE "server_start_queue" (
+  "id" TEXT NOT NULL,
+  "serverId" TEXT NOT NULL,
+  "nodeId" TEXT NOT NULL,
+  "status" "StartQueueStatus" NOT NULL DEFAULT 'QUEUED',
+  "priority" INTEGER NOT NULL DEFAULT 0,
+  "correlationId" TEXT,
+  "requestedAt" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  "expiresAt" TIMESTAMPTZ(3),
+  "createdAt" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  "updatedAt" TIMESTAMPTZ(3) NOT NULL,
+
+  CONSTRAINT "server_start_queue_pkey" PRIMARY KEY ("id")
+);
+
+CREATE UNIQUE INDEX "server_start_queue_serverId_key" ON "server_start_queue"("serverId");
+CREATE INDEX "server_start_queue_status_priority_requestedAt_idx" ON "server_start_queue"("status", "priority", "requestedAt");
+CREATE INDEX "server_start_queue_nodeId_idx" ON "server_start_queue"("nodeId");
+
+ALTER TABLE "server_start_queue"
+  ADD CONSTRAINT "server_start_queue_serverId_fkey"
+  FOREIGN KEY ("serverId") REFERENCES "game_servers"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+ALTER TABLE "server_start_queue"
+  ADD CONSTRAINT "server_start_queue_nodeId_fkey"
+  FOREIGN KEY ("nodeId") REFERENCES "game_nodes"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

packages/database/prisma/schema.prisma

@@ -34,11 +34,13 @@ enum GameServerStatus {
   PROVISIONING
   INSTALLING
   STOPPED
+  QUEUED
   STARTING
   RUNNING
   STOPPING
   BACKING_UP
   RESTORING
+  UNKNOWN
   ERROR
   DELETING
   DELETED
@@ -57,6 +59,13 @@ enum AllocationStatus {
   RELEASED
 }
 
+enum StartQueueStatus {
+  QUEUED
+  DISPATCHED
+  CANCELLED
+  EXPIRED
+}
+
 enum JobStatus {
   PENDING
   RUNNING
@@ -282,6 +291,7 @@ model GameServer {
   backupSchedule   BackupSchedule?
   worldUploads     WorldUpload[]
   installedAddons  InstalledAddon[]
+  startQueueEntry  ServerStartQueueEntry?
 
   @@index([userId])
   @@index([nodeId])
@@ -333,18 +343,22 @@ model GameNode {
   hostname            String
   status              GameNodeStatus @default(OFFLINE)
   maxServers          Int            @default(10)
+  maxRamMb            Int            @default(16384)
+  platformRamMb       Int            @default(2048)
   activeServers       Int            @default(0)
   lastHeartbeatAt     DateTime?      @db.Timestamptz(3)
   agentVersion        String?
   enrollmentTokenHash String?
   enrolledAt          DateTime?      @db.Timestamptz(3)
+  drainRequestedAt    DateTime?      @db.Timestamptz(3)
   metadata            Json?
   createdAt           DateTime       @default(now()) @db.Timestamptz(3)
   updatedAt           DateTime       @updatedAt @db.Timestamptz(3)
 
-  gameServers  GameServer[]
-  heartbeats   GameNodeHeartbeat[]
-  allocations  GameServerAllocation[]
+  gameServers       GameServer[]
+  heartbeats        GameNodeHeartbeat[]
+  allocations       GameServerAllocation[]
+  startQueueEntries ServerStartQueueEntry[]
 
   @@map("game_nodes")
 }
@@ -362,6 +376,26 @@ model GameNodeHeartbeat {
   @@map("game_node_heartbeats")
 }
 
+model ServerStartQueueEntry {
+  id            String           @id @default(uuid())
+  serverId      String           @unique
+  nodeId        String
+  status        StartQueueStatus @default(QUEUED)
+  priority      Int              @default(0)
+  correlationId String?
+  requestedAt   DateTime         @default(now()) @db.Timestamptz(3)
+  expiresAt     DateTime?        @db.Timestamptz(3)
+  createdAt     DateTime         @default(now()) @db.Timestamptz(3)
+  updatedAt     DateTime         @updatedAt @db.Timestamptz(3)
+
+  server GameServer @relation(fields: [serverId], references: [id], onDelete: Cascade)
+  node   GameNode   @relation(fields: [nodeId], references: [id])
+
+  @@index([status, priority, requestedAt])
+  @@index([nodeId])
+  @@map("server_start_queue")
+}
+
 model Plan {
   id           String   @id @default(uuid())
   name         String   @unique

packages/database/src/cli/seed-dev.ts

@@ -5,10 +5,15 @@ import { hashToken } from '@hexahost/auth';
 /** Stable dev node ID — must match NODE_ID in .env for local agent */
 export const DEV_NODE_ID = '00000000-0000-4000-8000-000000000001';
 
+export const DEV_NODE_ID_2 = '00000000-0000-4000-8000-000000000002';
+
 /** Shown once in seed output; hash stored in DB */
 export const DEV_NODE_ENROLLMENT_TOKEN =
   'local-dev-enrollment-token-change-me-32chars';
 
+export const DEV_NODE_2_ENROLLMENT_TOKEN =
+  'local-dev-node2-enrollment-token-32ch';
+
 async function main(): Promise<void> {
   const prisma = new PrismaClient();
 
@@ -38,19 +43,46 @@ async function main(): Promise<void> {
         hostname: 'localhost',
         status: 'OFFLINE',
         maxServers: 10,
+        maxRamMb: 16384,
+        platformRamMb: 2048,
         enrollmentTokenHash: hashToken(DEV_NODE_ENROLLMENT_TOKEN),
       },
       update: {
         name: 'local-dev',
         hostname: 'localhost',
+        maxRamMb: 16384,
+        platformRamMb: 2048,
         enrollmentTokenHash: hashToken(DEV_NODE_ENROLLMENT_TOKEN),
       },
     });
 
+    const devNode2 = await prisma.gameNode.upsert({
+      where: { id: DEV_NODE_ID_2 },
+      create: {
+        id: DEV_NODE_ID_2,
+        name: 'local-dev-2',
+        hostname: 'localhost',
+        status: 'OFFLINE',
+        maxServers: 10,
+        maxRamMb: 16384,
+        platformRamMb: 2048,
+        enrollmentTokenHash: hashToken(DEV_NODE_2_ENROLLMENT_TOKEN),
+      },
+      update: {
+        name: 'local-dev-2',
+        hostname: 'localhost',
+        maxRamMb: 16384,
+        platformRamMb: 2048,
+        enrollmentTokenHash: hashToken(DEV_NODE_2_ENROLLMENT_TOKEN),
+      },
+    });
+
     console.log('Seed complete:');
     console.log(`  Plan Free: ${freePlan.id}`);
     console.log(`  Dev node:  ${devNode.id} (${devNode.name})`);
+    console.log(`  Dev node 2: ${devNode2.id} (${devNode2.name})`);
     console.log(`  Enrollment token (set NODE_TOKEN): ${DEV_NODE_ENROLLMENT_TOKEN}`);
+    console.log(`  Node 2 enrollment token: ${DEV_NODE_2_ENROLLMENT_TOKEN}`);
   } finally {
     await prisma.$disconnect();
   }

packages/database/src/index.ts

@@ -19,6 +19,7 @@ export type {
   InstalledAddon,
   GameNode,
   GameNodeHeartbeat,
+  ServerStartQueueEntry,
   Plan,
   FeatureFlag,
   SystemSetting,

packages/scheduler/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@hexahost/scheduler",
+  "version": "0.0.0",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  },
+  "devDependencies": {
+    "@hexahost/typescript-config": "workspace:*",
+    "@types/node": "^22.10.2",
+    "typescript": "^5.7.3",
+    "vitest": "^3.0.8"
+  }
+}

packages/scheduler/src/capacity.ts

@@ -0,0 +1,98 @@
+import type { NodeCapacity, NodeSnapshot } from './types';
+
+export const HEARTBEAT_STALE_MS = 90_000;
+
+/** Nodes that accept new server allocations (provisioning). */
+export const ALLOCATION_ELIGIBLE_STATUSES = ['ONLINE'] as const;
+
+/** Nodes that can start already-provisioned servers. */
+export const START_ELIGIBLE_STATUSES = ['ONLINE', 'DRAINING'] as const;
+
+export function isHeartbeatFresh(
+  lastHeartbeatAt: Date | null,
+  now = Date.now(),
+): boolean {
+  if (!lastHeartbeatAt) {
+    return false;
+  }
+
+  return now - lastHeartbeatAt.getTime() <= HEARTBEAT_STALE_MS;
+}
+
+export function effectiveMaxRamMb(node: NodeSnapshot): number {
+  return Math.max(node.maxRamMb - node.platformRamMb, 0);
+}
+
+export function computeAvailableRamMb(node: NodeSnapshot): number {
+  const budget = effectiveMaxRamMb(node);
+
+  if (
+    node.memoryTotalBytes != null &&
+    node.memoryUsedBytes != null &&
+    node.memoryTotalBytes > 0
+  ) {
+    const hostFreeMb = Math.floor(
+      (node.memoryTotalBytes - node.memoryUsedBytes) / (1024 * 1024),
+    );
+    return Math.max(0, Math.min(budget - node.reservedRamMb, hostFreeMb));
+  }
+
+  return Math.max(0, budget - node.reservedRamMb);
+}
+
+export function computeAvailableSlots(node: NodeSnapshot): number {
+  const byMax = Math.max(0, node.maxServers - node.activeAllocationCount);
+  const byActive = Math.max(0, node.maxServers - node.activeServers);
+  return Math.min(byMax, byActive);
+}
+
+export function canAllocateOnNode(node: NodeSnapshot, requiredRamMb: number): boolean {
+  if (!ALLOCATION_ELIGIBLE_STATUSES.includes(node.status as (typeof ALLOCATION_ELIGIBLE_STATUSES)[number])) {
+    return false;
+  }
+
+  if (!isHeartbeatFresh(node.lastHeartbeatAt)) {
+    return false;
+  }
+
+  const capacity = evaluateNodeCapacity(node, requiredRamMb);
+  return capacity.schedulable;
+}
+
+export function canStartOnNode(node: NodeSnapshot, requiredRamMb: number): boolean {
+  if (!START_ELIGIBLE_STATUSES.includes(node.status as (typeof START_ELIGIBLE_STATUSES)[number])) {
+    return false;
+  }
+
+  if (!isHeartbeatFresh(node.lastHeartbeatAt)) {
+    return false;
+  }
+
+  const capacity = evaluateNodeCapacity(node, requiredRamMb);
+  return capacity.schedulable;
+}
+
+export function evaluateNodeCapacity(
+  node: NodeSnapshot,
+  requiredRamMb: number,
+): NodeCapacity {
+  const availableRamMb = computeAvailableRamMb(node);
+  const availableSlots = computeAvailableSlots(node);
+  const schedulable =
+    requiredRamMb > 0 &&
+    availableRamMb >= requiredRamMb &&
+    availableSlots > 0;
+
+  const ramHeadroom = availableRamMb - requiredRamMb;
+  const slotHeadroom = availableSlots - 1;
+  const score = schedulable
+    ? ramHeadroom * 10 + slotHeadroom * 100 - node.reservedRamMb
+    : -1;
+
+  return {
+    schedulable,
+    availableRamMb,
+    availableSlots,
+    score,
+  };
+}

packages/scheduler/src/index.ts

@@ -0,0 +1,19 @@
+export type { GameNodeStatus, NodeCapacity, NodeSnapshot } from './types';
+
+export {
+  ALLOCATION_ELIGIBLE_STATUSES,
+  START_ELIGIBLE_STATUSES,
+  HEARTBEAT_STALE_MS,
+  isHeartbeatFresh,
+  effectiveMaxRamMb,
+  computeAvailableRamMb,
+  computeAvailableSlots,
+  canAllocateOnNode,
+  canStartOnNode,
+  evaluateNodeCapacity,
+} from './capacity';
+
+export {
+  selectNodeForAllocation,
+  findNodeSnapshot,
+} from './select-node';

packages/scheduler/src/scheduling.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  canAllocateOnNode,
+  canStartOnNode,
+  selectNodeForAllocation,
+} from './index';
+import type { NodeSnapshot } from './types';
+
+function node(partial: Partial<NodeSnapshot> & Pick<NodeSnapshot, 'id'>): NodeSnapshot {
+  return {
+    status: 'ONLINE',
+    maxServers: 10,
+    maxRamMb: 16384,
+    platformRamMb: 2048,
+    activeServers: 0,
+    lastHeartbeatAt: new Date(),
+    reservedRamMb: 0,
+    activeAllocationCount: 0,
+    ...partial,
+  };
+}
+
+describe('scheduler capacity', () => {
+  it('rejects maintenance and draining nodes for new allocations', () => {
+    const maintenance = node({ id: 'a', status: 'MAINTENANCE' });
+    const draining = node({ id: 'b', status: 'DRAINING' });
+
+    expect(canAllocateOnNode(maintenance, 1024)).toBe(false);
+    expect(canAllocateOnNode(draining, 1024)).toBe(false);
+    expect(canStartOnNode(draining, 1024)).toBe(true);
+  });
+
+  it('picks the least loaded node for placement', () => {
+    const lightlyLoaded = node({ id: 'light', reservedRamMb: 1024 });
+    const heavilyLoaded = node({ id: 'heavy', reservedRamMb: 8192 });
+
+    const selected = selectNodeForAllocation(
+      [heavilyLoaded, lightlyLoaded],
+      2048,
+    );
+
+    expect(selected?.id).toBe('light');
+  });
+
+  it('returns null when no node has enough RAM', () => {
+    const saturated = node({
+      id: 'full',
+      reservedRamMb: 14000,
+      maxRamMb: 16384,
+      platformRamMb: 2048,
+    });
+
+    expect(selectNodeForAllocation([saturated], 4096)).toBeNull();
+  });
+});

packages/scheduler/src/select-node.ts

@@ -0,0 +1,30 @@
+import { canAllocateOnNode, evaluateNodeCapacity } from './capacity';
+import type { NodeSnapshot } from './types';
+
+export function selectNodeForAllocation(
+  nodes: NodeSnapshot[],
+  requiredRamMb: number,
+): NodeSnapshot | null {
+  const candidates = nodes
+    .filter((node) => canAllocateOnNode(node, requiredRamMb))
+    .map((node) => ({
+      node,
+      capacity: evaluateNodeCapacity(node, requiredRamMb),
+    }))
+    .sort((left, right) => {
+      if (right.capacity.score !== left.capacity.score) {
+        return right.capacity.score - left.capacity.score;
+      }
+
+      return left.node.reservedRamMb - right.node.reservedRamMb;
+    });
+
+  return candidates[0]?.node ?? null;
+}
+
+export function findNodeSnapshot(
+  nodes: NodeSnapshot[],
+  nodeId: string,
+): NodeSnapshot | null {
+  return nodes.find((node) => node.id === nodeId) ?? null;
+}

packages/scheduler/src/types.ts

@@ -0,0 +1,27 @@
+export type GameNodeStatus =
+  | 'ONLINE'
+  | 'OFFLINE'
+  | 'MAINTENANCE'
+  | 'DRAINING'
+  | 'UNREACHABLE';
+
+export interface NodeSnapshot {
+  id: string;
+  status: GameNodeStatus;
+  maxServers: number;
+  maxRamMb: number;
+  platformRamMb: number;
+  activeServers: number;
+  lastHeartbeatAt: Date | null;
+  memoryTotalBytes?: number | null;
+  memoryUsedBytes?: number | null;
+  reservedRamMb: number;
+  activeAllocationCount: number;
+}
+
+export interface NodeCapacity {
+  schedulable: boolean;
+  availableRamMb: number;
+  availableSlots: number;
+  score: number;
+}

packages/scheduler/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "@hexahost/typescript-config/base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}