Phase6
Some checks failed
CI / Node — lint, typecheck, test, build (push) Failing after 9s
CI / Go — node-agent tests (push) Failing after 8s
CI / Go — edge-gateway build (push) Successful in 13s

This commit is contained in:
smueller
2026-06-26 13:11:28 +02:00
parent d29a02f2a4
commit 49a98ee31d
81 changed files with 1462 additions and 53 deletions

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
{"hash":"0dddfbd60b2cad9b","duration":15192,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"0c8ee2dc7233a2788afb738a05f16058a7b7070e18c6759c9b915f110862719c"}

BIN
.turbo/cache/0dddfbd60b2cad9b.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"packages/scheduler/dist/types.d.ts.map":{"size":664,"mtime_nanos":1782472054358321300,"mode":420,"is_dir":false},"packages/scheduler/dist/types.js.map":{"size":102,"mtime_nanos":1782472054343782700,"mode":420,"is_dir":false},"packages/scheduler/dist/types.js":{"size":110,"mtime_nanos":1782472054345425800,"mode":420,"is_dir":false},"packages/scheduler/dist/select-node.d.ts.map":{"size":347,"mtime_nanos":1782472054430618300,"mode":420,"is_dir":false},"packages/scheduler/.turbo/turbo-build.log":{"size":111,"mtime_nanos":1782472054549437300,"mode":420,"is_dir":false},"packages/scheduler/dist/select-node.js":{"size":944,"mtime_nanos":1782472054426748400,"mode":420,"is_dir":false},"packages/scheduler/dist":{"size":0,"mtime_nanos":0,"mode":0,"is_dir":true},"packages/scheduler/dist/scheduling.test.js.map":{"size":1650,"mtime_nanos":1782472054456735900,"mode":420,"is_dir":false},"packages/scheduler/dist/index.d.ts.map":{"size":378,"mtime_nanos":1782472054446698300,"mode":420,"is_dir":false},"packages/scheduler/dist/scheduling.test.d.ts.map":{"size":124,"mtime_nanos":1782472054459826800,"mode":420,"is_dir":false},"packages/scheduler/dist/select-node.d.ts":{"size":304,"mtime_nanos":1782472054431746800,"mode":420,"is_dir":false},"packages/scheduler/dist/types.d.ts":{"size":615,"mtime_nanos":1782472054360997700,"mode":420,"is_dir":false},"packages/scheduler/dist/capacity.js.map":{"size":2635,"mtime_nanos":1782472054393945100,"mode":420,"is_dir":false},"packages/scheduler/dist/index.js":{"size":2244,"mtime_nanos":1782472054443704200,"mode":420,"is_dir":false},"packages/scheduler/dist/index.d.ts":{"size":431,"mtime_nanos":1782472054447576100,"mode":420,"is_dir":false},"packages/scheduler/dist/capacity.js":{"size":3044,"mtime_nanos":1782472054395645800,"mode":420,"is_dir":false},"packages/scheduler/dist/index.js.map":{"size":414,"mtime_nanos":1782472054442363300,"mode":420,"is_dir":false},"packages/scheduler/dist/scheduling.test.d.ts":{"size":56,"mtime_nanos":1782472054464109200,"mode":420,"is_dir":false},"packages/scheduler/dist/scheduling.test.js":{"size":1778,"mtime_nanos":1782472054457833000,"mode":420,"is_dir":false},"packages/scheduler/dist/capacity.d.ts.map":{"size":766,"mtime_nanos":1782472054413025900,"mode":420,"is_dir":false},"packages/scheduler/dist/capacity.d.ts":{"size":1024,"mtime_nanos":1782472054414689000,"mode":420,"is_dir":false},"packages/scheduler/dist/select-node.js.map":{"size":939,"mtime_nanos":1782472054424633100,"mode":420,"is_dir":false}},"order":["packages/scheduler/.turbo/turbo-build.log","packages/scheduler/dist","packages/scheduler/dist/capacity.d.ts","packages/scheduler/dist/capacity.d.ts.map","packages/scheduler/dist/capacity.js","packages/scheduler/dist/capacity.js.map","packages/scheduler/dist/index.d.ts","packages/scheduler/dist/index.d.ts.map","packages/scheduler/dist/index.js","packages/scheduler/dist/index.js.map","packages/scheduler/dist/scheduling.test.d.ts","packages/scheduler/dist/scheduling.test.d.ts.map","packages/scheduler/dist/scheduling.test.js","packages/scheduler/dist/scheduling.test.js.map","packages/scheduler/dist/select-node.d.ts","packages/scheduler/dist/select-node.d.ts.map","packages/scheduler/dist/select-node.js","packages/scheduler/dist/select-node.js.map","packages/scheduler/dist/types.d.ts","packages/scheduler/dist/types.d.ts.map","packages/scheduler/dist/types.js","packages/scheduler/dist/types.js.map"]}

View File

@@ -0,0 +1 @@
{"hash":"20fcc3444a9e3785","duration":3159,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"0c8ee2dc7233a2788afb738a05f16058a7b7070e18c6759c9b915f110862719c"}

BIN
.turbo/cache/20fcc3444a9e3785.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"packages/database/dist/cli/bootstrap-admin.d.ts.map":{"size":131,"mtime_nanos":1782466100085644700,"mode":420,"is_dir":false},"packages/database/dist/cli/bootstrap-admin.d.ts":{"size":56,"mtime_nanos":1782466100087324100,"mode":420,"is_dir":false},"packages/database/dist/cli/bootstrap-admin.js.map":{"size":3941,"mtime_nanos":1782466100078293500,"mode":420,"is_dir":false},"packages/database/dist/index.js":{"size":962,"mtime_nanos":1782472059551844200,"mode":420,"is_dir":false},"packages/database/dist/index.js.map":{"size":617,"mtime_nanos":1782472059550520000,"mode":420,"is_dir":false},"packages/database/dist/cli/bootstrap-admin.js":{"size":4338,"mtime_nanos":1782466100081295500,"mode":420,"is_dir":false},"packages/database/dist/index.d.ts":{"size":778,"mtime_nanos":1782472059557653200,"mode":420,"is_dir":false},"packages/database/dist/cli":{"size":0,"mtime_nanos":0,"mode":0,"is_dir":true},"packages/database/dist":{"size":0,"mtime_nanos":0,"mode":0,"is_dir":true},"packages/database/.turbo/turbo-build.log":{"size":509,"mtime_nanos":1782472119448413000,"mode":420,"is_dir":false},"packages/database/dist/index.d.ts.map":{"size":569,"mtime_nanos":1782472059555875000,"mode":420,"is_dir":false}},"order":["packages/database/.turbo/turbo-build.log","packages/database/dist","packages/database/dist/cli","packages/database/dist/cli/bootstrap-admin.d.ts","packages/database/dist/cli/bootstrap-admin.d.ts.map","packages/database/dist/cli/bootstrap-admin.js","packages/database/dist/cli/bootstrap-admin.js.map","packages/database/dist/index.d.ts","packages/database/dist/index.d.ts.map","packages/database/dist/index.js","packages/database/dist/index.js.map"]}

View File

@@ -0,0 +1 @@
{"hash":"450c6859a560198c","duration":6383,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/450c6859a560198c.tar.zst vendored Normal file

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
{"hash":"4900ef9a5ab4e8e4","duration":37983,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"0c8ee2dc7233a2788afb738a05f16058a7b7070e18c6759c9b915f110862719c"}

BIN
.turbo/cache/4900ef9a5ab4e8e4.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"apps/web/.turbo/turbo-test.log":{"size":116,"mtime_nanos":1782472114334476800,"mode":420,"is_dir":false}},"order":["apps/web/.turbo/turbo-test.log"]}

View File

@@ -0,0 +1 @@
{"hash":"4f47dc608cbdbcfa","duration":1270,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/4f47dc608cbdbcfa.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"packages/database/dist":{"size":0,"mtime_nanos":0,"mode":0,"is_dir":true},"packages/database/dist/cli/bootstrap-admin.d.ts":{"size":56,"mtime_nanos":1782466100087324100,"mode":420,"is_dir":false},"packages/database/dist/index.d.ts":{"size":778,"mtime_nanos":1782472059557653200,"mode":420,"is_dir":false},"packages/database/dist/index.js":{"size":962,"mtime_nanos":1782472059551844200,"mode":420,"is_dir":false},"packages/database/dist/cli":{"size":0,"mtime_nanos":0,"mode":0,"is_dir":true},"packages/database/dist/index.d.ts.map":{"size":569,"mtime_nanos":1782472059555875000,"mode":420,"is_dir":false},"packages/database/dist/cli/bootstrap-admin.js.map":{"size":3941,"mtime_nanos":1782466100078293500,"mode":420,"is_dir":false},"packages/database/dist/cli/bootstrap-admin.d.ts.map":{"size":131,"mtime_nanos":1782466100085644700,"mode":420,"is_dir":false},"packages/database/dist/cli/bootstrap-admin.js":{"size":4338,"mtime_nanos":1782466100081295500,"mode":420,"is_dir":false},"packages/database/dist/index.js.map":{"size":617,"mtime_nanos":1782472059550520000,"mode":420,"is_dir":false},"packages/database/.turbo/turbo-build.log":{"size":552,"mtime_nanos":1782472059680260700,"mode":420,"is_dir":false}},"order":["packages/database/.turbo/turbo-build.log","packages/database/dist","packages/database/dist/cli","packages/database/dist/cli/bootstrap-admin.d.ts","packages/database/dist/cli/bootstrap-admin.d.ts.map","packages/database/dist/cli/bootstrap-admin.js","packages/database/dist/cli/bootstrap-admin.js.map","packages/database/dist/index.d.ts","packages/database/dist/index.d.ts.map","packages/database/dist/index.js","packages/database/dist/index.js.map"]}

View File

@@ -0,0 +1 @@
{"hash":"550644223cc03927","duration":8280,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"0c8ee2dc7233a2788afb738a05f16058a7b7070e18c6759c9b915f110862719c"}

BIN
.turbo/cache/550644223cc03927.tar.zst vendored Normal file

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
{"hash":"5c5877c7aa16b614","duration":7685,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"0c8ee2dc7233a2788afb738a05f16058a7b7070e18c6759c9b915f110862719c"}

BIN
.turbo/cache/5c5877c7aa16b614.tar.zst vendored Normal file

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
{"hash":"68ebd47fd2a5be9d","duration":1954,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/68ebd47fd2a5be9d.tar.zst vendored Normal file

Binary file not shown.

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1 @@
{"hash":"91ae97009f1f9ed2","duration":2934,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"0c8ee2dc7233a2788afb738a05f16058a7b7070e18c6759c9b915f110862719c"}

BIN
.turbo/cache/91ae97009f1f9ed2.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"apps/api/.turbo/turbo-test.log":{"size":3720,"mtime_nanos":1782472129339886300,"mode":420,"is_dir":false}},"order":["apps/api/.turbo/turbo-test.log"]}

View File

@@ -0,0 +1 @@
{"hash":"9a2bc04224bb3a5e","duration":9870,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/9a2bc04224bb3a5e.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"packages/contracts/.turbo/turbo-test.log":{"size":459,"mtime_nanos":1782472116833592300,"mode":420,"is_dir":false}},"order":["packages/contracts/.turbo/turbo-test.log"]}

View File

@@ -0,0 +1 @@
{"hash":"b9c868a33b024c82","duration":3772,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/b9c868a33b024c82.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"packages/scheduler/.turbo/turbo-test.log":{"size":425,"mtime_nanos":1782472116819604200,"mode":420,"is_dir":false}},"order":["packages/scheduler/.turbo/turbo-test.log"]}

View File

@@ -0,0 +1 @@
{"hash":"bf5522815be54147","duration":3772,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/bf5522815be54147.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1 @@
{"files":{"apps/worker/.turbo/turbo-test.log":{"size":834,"mtime_nanos":1782472120797404300,"mode":420,"is_dir":false}},"order":["apps/worker/.turbo/turbo-test.log"]}

View File

@@ -0,0 +1 @@
{"hash":"f622c19665119e3b","duration":1330,"sha":"d29a02f2a49fb5a2d17152fe837c5668c3df3cee","dirty_hash":"74f5f535c2759fdeb072426ebf3702a72586b5a92ba07d3e028238e5985f70f8"}

BIN
.turbo/cache/f622c19665119e3b.tar.zst vendored Normal file

Binary file not shown.

View File

@@ -92,7 +92,7 @@ make build
## Aktueller Stand ## Aktueller Stand
**Phase 5** (Software & Add-ons) — abgeschlossen. **Phase 6** (Multi-Node & Queue) — abgeschlossen.
Siehe [docs/IMPLEMENTATION_STATUS.md](docs/IMPLEMENTATION_STATUS.md) für Details. Siehe [docs/IMPLEMENTATION_STATUS.md](docs/IMPLEMENTATION_STATUS.md) für Details.

View File

@@ -20,6 +20,7 @@
"@hexahost/config": "workspace:*", "@hexahost/config": "workspace:*",
"@hexahost/contracts": "workspace:*", "@hexahost/contracts": "workspace:*",
"@hexahost/database": "workspace:*", "@hexahost/database": "workspace:*",
"@hexahost/scheduler": "workspace:*",
"@hexahost/storage": "workspace:*", "@hexahost/storage": "workspace:*",
"@nestjs/common": "^11.0.7", "@nestjs/common": "^11.0.7",
"@nestjs/core": "^11.0.7", "@nestjs/core": "^11.0.7",

View File

@@ -1,14 +1,19 @@
import { import {
Body, Body,
Controller, Controller,
Get,
HttpCode, HttpCode,
HttpStatus, HttpStatus,
Param,
ParseUUIDPipe,
Post, Post,
UseGuards, UseGuards,
} from '@nestjs/common'; } from '@nestjs/common';
import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger'; import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger';
import { z } from 'zod'; import { z } from 'zod';
import type { NodeListResponse, NodeSummary } from '@hexahost/contracts';
import { CurrentUser } from '../auth/decorators/current-user.decorator'; import { CurrentUser } from '../auth/decorators/current-user.decorator';
import { SessionGuard } from '../auth/guards/session.guard'; import { SessionGuard } from '../auth/guards/session.guard';
import { ZodValidationPipe } from '../common/pipes/zod-validation.pipe'; import { ZodValidationPipe } from '../common/pipes/zod-validation.pipe';
@@ -20,6 +25,12 @@ const createNodeRequestSchema = z.object({
name: z.string().min(1).max(64), name: z.string().min(1).max(64),
hostname: z.string().min(1).max(255), hostname: z.string().min(1).max(255),
maxServers: z.number().int().min(1).max(1000).optional(), maxServers: z.number().int().min(1).max(1000).optional(),
maxRamMb: z.number().int().min(2048).max(1048576).optional(),
platformRamMb: z.number().int().min(512).max(65536).optional(),
});
const maintenanceRequestSchema = z.object({
enabled: z.boolean(),
}); });
@ApiTags('admin') @ApiTags('admin')
@@ -28,6 +39,13 @@ const createNodeRequestSchema = z.object({
export class AdminNodesController { export class AdminNodesController {
constructor(private readonly adminNodesService: AdminNodesService) {} constructor(private readonly adminNodesService: AdminNodesService) {}
@Get()
@ApiOperation({ summary: 'List game nodes (super admin only)' })
list(@CurrentUser() user: { roles: string[] }): Promise<NodeListResponse> {
this.adminNodesService.assertSuperAdmin(user.roles);
return this.adminNodesService.listNodes();
}
@Post() @Post()
@HttpCode(HttpStatus.CREATED) @HttpCode(HttpStatus.CREATED)
@ApiOperation({ summary: 'Register a new game node (super admin only)' }) @ApiOperation({ summary: 'Register a new game node (super admin only)' })
@@ -42,4 +60,28 @@ export class AdminNodesController {
body as Parameters<AdminNodesService['createNode']>[0], body as Parameters<AdminNodesService['createNode']>[0],
); );
} }
@Post(':id/drain')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: 'Drain a game node (super admin only)' })
drain(
@CurrentUser() user: { roles: string[] },
@Param('id', ParseUUIDPipe) id: string,
): Promise<NodeSummary> {
this.adminNodesService.assertSuperAdmin(user.roles);
return this.adminNodesService.requestDrain(id);
}
@Post(':id/maintenance')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: 'Toggle maintenance mode for a node (super admin only)' })
maintenance(
@CurrentUser() user: { roles: string[] },
@Param('id', ParseUUIDPipe) id: string,
@Body(new ZodValidationPipe(maintenanceRequestSchema)) body: unknown,
): Promise<NodeSummary> {
this.adminNodesService.assertSuperAdmin(user.roles);
const input = body as z.infer<typeof maintenanceRequestSchema>;
return this.adminNodesService.setMaintenance(id, input.enabled);
}
} }

View File

@@ -1,17 +1,27 @@
import { ForbiddenException, Injectable } from '@nestjs/common'; import {
ConflictException,
ForbiddenException,
Injectable,
NotFoundException,
} from '@nestjs/common';
import { import {
generateSecureToken, generateSecureToken,
hashToken, hashToken,
UserRole, UserRole,
} from '@hexahost/auth'; } from '@hexahost/auth';
import type { NodeListResponse, NodeSummary } from '@hexahost/contracts';
import { computeAvailableRamMb } from '@hexahost/scheduler';
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { loadNodeSnapshots } from '../scheduling/node-snapshots';
export interface CreateNodeInput { export interface CreateNodeInput {
name: string; name: string;
hostname: string; hostname: string;
maxServers?: number; maxServers?: number;
maxRamMb?: number;
platformRamMb?: number;
} }
export interface CreateNodeResponse { export interface CreateNodeResponse {
@@ -21,6 +31,7 @@ export interface CreateNodeResponse {
hostname: string; hostname: string;
status: string; status: string;
maxServers: number; maxServers: number;
maxRamMb: number;
createdAt: string; createdAt: string;
}; };
enrollmentToken: string; enrollmentToken: string;
@@ -36,6 +47,19 @@ export class AdminNodesService {
} }
} }
async listNodes(): Promise<NodeListResponse> {
const snapshots = await loadNodeSnapshots();
const nodes = await this.prisma.gameNode.findMany({
orderBy: { createdAt: 'asc' },
});
const snapshotById = new Map(snapshots.map((snapshot) => [snapshot.id, snapshot]));
return {
nodes: nodes.map((node) => this.toSummary(node, snapshotById.get(node.id))),
};
}
async createNode(input: CreateNodeInput): Promise<CreateNodeResponse> { async createNode(input: CreateNodeInput): Promise<CreateNodeResponse> {
const enrollmentToken = generateSecureToken(); const enrollmentToken = generateSecureToken();
@@ -44,6 +68,8 @@ export class AdminNodesService {
name: input.name, name: input.name,
hostname: input.hostname, hostname: input.hostname,
maxServers: input.maxServers ?? 10, maxServers: input.maxServers ?? 10,
maxRamMb: input.maxRamMb ?? 16384,
platformRamMb: input.platformRamMb ?? 2048,
enrollmentTokenHash: hashToken(enrollmentToken), enrollmentTokenHash: hashToken(enrollmentToken),
}, },
}); });
@@ -55,9 +81,109 @@ export class AdminNodesService {
hostname: node.hostname, hostname: node.hostname,
status: node.status, status: node.status,
maxServers: node.maxServers, maxServers: node.maxServers,
maxRamMb: node.maxRamMb,
createdAt: node.createdAt.toISOString(), createdAt: node.createdAt.toISOString(),
}, },
enrollmentToken, enrollmentToken,
}; };
} }
async requestDrain(nodeId: string): Promise<NodeSummary> {
const node = await this.requireNode(nodeId);
if (node.status === 'DRAINING' || node.status === 'MAINTENANCE') {
throw new ConflictException('Node is already draining or in maintenance');
}
const updated = await this.prisma.gameNode.update({
where: { id: nodeId },
data: {
status: 'DRAINING',
drainRequestedAt: new Date(),
},
});
const snapshot = (await loadNodeSnapshots([nodeId]))[0];
return this.toSummary(updated, snapshot);
}
async setMaintenance(nodeId: string, enabled: boolean): Promise<NodeSummary> {
const node = await this.requireNode(nodeId);
if (enabled && node.status === 'MAINTENANCE') {
const snapshot = (await loadNodeSnapshots([nodeId]))[0];
return this.toSummary(node, snapshot);
}
const updated = await this.prisma.gameNode.update({
where: { id: nodeId },
data: {
status: enabled ? 'MAINTENANCE' : 'OFFLINE',
drainRequestedAt: null,
},
});
const snapshot = (await loadNodeSnapshots([nodeId]))[0];
return this.toSummary(updated, snapshot);
}
private async requireNode(nodeId: string) {
const node = await this.prisma.gameNode.findUnique({ where: { id: nodeId } });
if (!node) {
throw new NotFoundException('Node not found');
}
return node;
}
private toSummary(
node: {
id: string;
name: string;
hostname: string;
status: NodeSummary['status'];
maxServers: number;
maxRamMb: number;
platformRamMb: number;
activeServers: number;
lastHeartbeatAt: Date | null;
agentVersion: string | null;
drainRequestedAt: Date | null;
createdAt: Date;
},
snapshot?: {
reservedRamMb: number;
},
): NodeSummary {
const reservedRamMb = snapshot?.reservedRamMb ?? 0;
const availableRamMb = snapshot
? computeAvailableRamMb({
id: node.id,
status: node.status,
maxServers: node.maxServers,
maxRamMb: node.maxRamMb,
platformRamMb: node.platformRamMb,
activeServers: node.activeServers,
lastHeartbeatAt: node.lastHeartbeatAt,
reservedRamMb,
activeAllocationCount: 0,
})
: Math.max(0, node.maxRamMb - node.platformRamMb - reservedRamMb);
return {
id: node.id,
name: node.name,
hostname: node.hostname,
status: node.status,
maxServers: node.maxServers,
maxRamMb: node.maxRamMb,
platformRamMb: node.platformRamMb,
activeServers: node.activeServers,
reservedRamMb,
availableRamMb,
lastHeartbeatAt: node.lastHeartbeatAt?.toISOString() ?? null,
agentVersion: node.agentVersion,
drainRequestedAt: node.drainRequestedAt?.toISOString() ?? null,
createdAt: node.createdAt.toISOString(),
};
}
} }

View File

@@ -110,10 +110,18 @@ export class NodesService {
} }
async markNodeOnline(nodeId: string, agentVersion?: string): Promise<void> { async markNodeOnline(nodeId: string, agentVersion?: string): Promise<void> {
const node = await this.prisma.gameNode.findUnique({ where: { id: nodeId } });
const nextStatus =
node?.status === 'UNREACHABLE' || node?.status === 'OFFLINE'
? 'ONLINE'
: node?.status === 'MAINTENANCE' || node?.status === 'DRAINING'
? node.status
: 'ONLINE';
await this.prisma.gameNode.update({ await this.prisma.gameNode.update({
where: { id: nodeId }, where: { id: nodeId },
data: { data: {
status: 'ONLINE', status: nextStatus,
lastHeartbeatAt: new Date(), lastHeartbeatAt: new Date(),
enrolledAt: new Date(), enrolledAt: new Date(),
agentVersion, agentVersion,
@@ -172,17 +180,35 @@ export class NodesService {
payload?: Record<string, unknown>, payload?: Record<string, unknown>,
): Promise<void> { ): Promise<void> {
const now = new Date(); const now = new Date();
const node = await this.prisma.gameNode.findUnique({ where: { id: nodeId } });
const nextStatus =
node?.status === 'UNREACHABLE'
? 'ONLINE'
: node?.status === 'MAINTENANCE' || node?.status === 'DRAINING'
? node.status
: 'ONLINE';
await this.prisma.$transaction([ await this.prisma.$transaction([
this.prisma.gameNode.update({ this.prisma.gameNode.update({
where: { id: nodeId }, where: { id: nodeId },
data: { data: {
lastHeartbeatAt: now, lastHeartbeatAt: now,
status: 'ONLINE', status: nextStatus,
agentVersion: agentVersion:
typeof payload?.['agentVersion'] === 'string' typeof payload?.['agentVersion'] === 'string'
? payload['agentVersion'] ? payload['agentVersion']
: undefined, : undefined,
metadata: payload
? ({
cpuUsagePercent: payload['cpuUsagePercent'],
memoryUsedBytes: payload['memoryUsedBytes'],
memoryTotalBytes: payload['memoryTotalBytes'],
diskUsedBytes: payload['diskUsedBytes'],
diskTotalBytes: payload['diskTotalBytes'],
runningServers: payload['runningServers'],
} as Prisma.InputJsonValue)
: undefined,
}, },
}), }),
this.prisma.gameNodeHeartbeat.create({ this.prisma.gameNodeHeartbeat.create({

View File

@@ -0,0 +1,59 @@
import { prisma } from '@hexahost/database';
import type { NodeSnapshot } from '@hexahost/scheduler';
function readMemoryField(
payload: Record<string, unknown> | null | undefined,
key: 'memoryTotalBytes' | 'memoryUsedBytes',
): number | null {
const value = payload?.[key];
return typeof value === 'number' ? value : null;
}
export async function loadNodeSnapshots(
nodeIds?: string[],
): Promise<NodeSnapshot[]> {
const nodes = await prisma.gameNode.findMany({
where: nodeIds ? { id: { in: nodeIds } } : undefined,
include: {
allocations: {
where: { status: 'ACTIVE' },
select: { ramMbReserved: true },
},
heartbeats: {
orderBy: { createdAt: 'desc' },
take: 1,
select: { payload: true },
},
},
});
return nodes.map((node) => {
const heartbeatPayload = node.heartbeats[0]?.payload as
| Record<string, unknown>
| undefined;
const reservedRamMb = node.allocations.reduce(
(sum, allocation) => sum + allocation.ramMbReserved,
0,
);
return {
id: node.id,
status: node.status,
maxServers: node.maxServers,
maxRamMb: node.maxRamMb,
platformRamMb: node.platformRamMb,
activeServers: node.activeServers,
lastHeartbeatAt: node.lastHeartbeatAt,
memoryTotalBytes: readMemoryField(heartbeatPayload, 'memoryTotalBytes'),
memoryUsedBytes: readMemoryField(heartbeatPayload, 'memoryUsedBytes'),
reservedRamMb,
activeAllocationCount: node.allocations.length,
};
});
}
export async function loadNodeSnapshot(nodeId: string): Promise<NodeSnapshot | null> {
const snapshots = await loadNodeSnapshots([nodeId]);
return snapshots[0] ?? null;
}

View File

@@ -0,0 +1,12 @@
import { Module } from '@nestjs/common';
import { PrismaModule } from '../prisma/prisma.module';
import { SchedulingService } from './scheduling.service';
@Module({
imports: [PrismaModule],
providers: [SchedulingService],
exports: [SchedulingService],
})
export class SchedulingModule {}

View File

@@ -0,0 +1,175 @@
import { randomUUID } from 'node:crypto';
import { Injectable } from '@nestjs/common';
import type { StartQueueResponse } from '@hexahost/contracts';
import type { GameServer } from '@hexahost/database';
import { canStartOnNode } from '@hexahost/scheduler';
import { PrismaService } from '../prisma/prisma.service';
import { loadNodeSnapshot } from './node-snapshots';
const START_QUEUE_EXPIRY_MS = 30 * 60 * 1000;
@Injectable()
export class SchedulingService {
constructor(private readonly prisma: PrismaService) {}
async canStartServer(server: GameServer): Promise<boolean> {
if (!server.nodeId) {
return false;
}
const snapshot = await loadNodeSnapshot(server.nodeId);
return snapshot ? canStartOnNode(snapshot, server.ramMb) : false;
}
async enqueueStart(
server: GameServer,
actorId: string,
correlationId: string,
): Promise<{ server: GameServer; queuePosition: number }> {
const expiresAt = new Date(Date.now() + START_QUEUE_EXPIRY_MS);
const updated = await this.prisma.$transaction(async (tx) => {
await tx.serverStartQueueEntry.upsert({
where: { serverId: server.id },
create: {
serverId: server.id,
nodeId: server.nodeId!,
status: 'QUEUED',
correlationId,
expiresAt,
},
update: {
status: 'QUEUED',
correlationId,
expiresAt,
requestedAt: new Date(),
},
});
const next = await tx.gameServer.update({
where: { id: server.id },
data: {
status: 'QUEUED',
version: { increment: 1 },
},
});
await tx.gameServerStateTransition.create({
data: {
gameServerId: server.id,
fromStatus: server.status,
toStatus: 'QUEUED',
reason: 'Start queued due to insufficient node capacity',
actorId,
correlationId,
},
});
return next;
});
const queuePosition = await this.getQueuePosition(server.id);
return { server: updated, queuePosition };
}
async getQueuePosition(serverId: string): Promise<number> {
const entry = await this.prisma.serverStartQueueEntry.findUnique({
where: { serverId },
});
if (!entry || entry.status !== 'QUEUED') {
return 0;
}
const ahead = await this.prisma.serverStartQueueEntry.count({
where: {
status: 'QUEUED',
OR: [
{ priority: { gt: entry.priority } },
{
priority: entry.priority,
requestedAt: { lt: entry.requestedAt },
},
],
},
});
return ahead + 1;
}
async getStartQueue(userId: string, serverId: string): Promise<StartQueueResponse> {
const server = await this.prisma.gameServer.findFirst({
where: { id: serverId, userId },
});
if (!server) {
return { entry: null };
}
const entry = await this.prisma.serverStartQueueEntry.findUnique({
where: { serverId },
});
if (!entry || entry.status !== 'QUEUED') {
return { entry: null };
}
const position = await this.getQueuePosition(serverId);
return {
entry: {
serverId: entry.serverId,
nodeId: entry.nodeId,
status: entry.status,
priority: entry.priority,
position,
requestedAt: entry.requestedAt.toISOString(),
expiresAt: entry.expiresAt?.toISOString() ?? null,
},
};
}
async cancelQueuedStart(userId: string, serverId: string): Promise<GameServer> {
const server = await this.prisma.gameServer.findFirst({
where: { id: serverId, userId },
});
if (!server) {
throw new Error('Server not found');
}
if (server.status !== 'QUEUED') {
return server;
}
return this.prisma.$transaction(async (tx) => {
await tx.serverStartQueueEntry.updateMany({
where: { serverId, status: 'QUEUED' },
data: { status: 'CANCELLED' },
});
const updated = await tx.gameServer.update({
where: { id: serverId },
data: { status: 'STOPPED', version: { increment: 1 } },
});
await tx.gameServerStateTransition.create({
data: {
gameServerId: serverId,
fromStatus: 'QUEUED',
toStatus: 'STOPPED',
reason: 'Queued start cancelled by user',
actorId: userId,
correlationId: randomUUID(),
},
});
return updated;
});
}
}

View File

@@ -7,13 +7,15 @@ const ALLOWED_TRANSITIONS: Record<GameServerStatus, GameServerStatus[]> = {
DRAFT: ['PROVISIONING', 'DELETING'], DRAFT: ['PROVISIONING', 'DELETING'],
PROVISIONING: ['INSTALLING', 'ERROR', 'DELETING'], PROVISIONING: ['INSTALLING', 'ERROR', 'DELETING'],
INSTALLING: ['STOPPED', 'ERROR', 'DELETING'], INSTALLING: ['STOPPED', 'ERROR', 'DELETING'],
STOPPED: ['STARTING', 'PROVISIONING', 'BACKING_UP', 'RESTORING', 'DELETING'], STOPPED: ['STARTING', 'QUEUED', 'PROVISIONING', 'BACKING_UP', 'RESTORING', 'DELETING'],
STARTING: ['RUNNING', 'STOPPED', 'ERROR', 'DELETING'], QUEUED: ['STARTING', 'STOPPED', 'DELETING'],
RUNNING: ['STOPPING', 'BACKING_UP', 'ERROR', 'DELETING'], STARTING: ['RUNNING', 'STOPPED', 'ERROR', 'UNKNOWN', 'DELETING'],
STOPPING: ['STOPPED', 'ERROR', 'DELETING'], RUNNING: ['STOPPING', 'BACKING_UP', 'ERROR', 'UNKNOWN', 'DELETING'],
STOPPING: ['STOPPED', 'ERROR', 'UNKNOWN', 'DELETING'],
BACKING_UP: ['RUNNING', 'STOPPED', 'ERROR'], BACKING_UP: ['RUNNING', 'STOPPED', 'ERROR'],
RESTORING: ['STOPPED', 'ERROR'], RESTORING: ['STOPPED', 'ERROR'],
ERROR: ['STOPPED', 'STARTING', 'PROVISIONING', 'DELETING'], UNKNOWN: ['STOPPED', 'STARTING', 'ERROR', 'DELETING'],
ERROR: ['STOPPED', 'STARTING', 'QUEUED', 'PROVISIONING', 'DELETING'],
DELETING: ['DELETED'], DELETING: ['DELETED'],
DELETED: [], DELETED: [],
}; };
@@ -55,6 +57,8 @@ export class ServerStateService {
case 'ERROR': case 'ERROR':
return 'PROVISIONING'; return 'PROVISIONING';
case 'STOPPED': case 'STOPPED':
case 'UNKNOWN':
case 'QUEUED':
return 'STARTING'; return 'STARTING';
default: default:
throw new BadRequestException( throw new BadRequestException(

View File

@@ -85,4 +85,23 @@ export class ServersController {
) { ) {
return this.serversService.restartServer(user.id, id, skip === 'true'); return this.serversService.restartServer(user.id, id, skip === 'true');
} }
@Get(':id/start-queue')
@ApiOperation({ summary: 'Get start queue status for a server' })
startQueue(
@CurrentUser() user: { id: string },
@Param('id', ParseUUIDPipe) id: string,
) {
return this.serversService.getStartQueue(user.id, id);
}
@Post(':id/cancel-queue')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: 'Cancel a queued server start' })
cancelQueue(
@CurrentUser() user: { id: string },
@Param('id', ParseUUIDPipe) id: string,
) {
return this.serversService.cancelQueuedStart(user.id, id);
}
} }

View File

@@ -8,6 +8,8 @@ import { AuthModule } from '../auth/auth.module';
import { NodeBridgeModule } from '../nodes/node-bridge.module'; import { NodeBridgeModule } from '../nodes/node-bridge.module';
import { AppConfigModule } from '../config/app-config.module'; import { AppConfigModule } from '../config/app-config.module';
import { SchedulingModule } from '../scheduling/scheduling.module';
import { ConsoleModule } from './console/console.module'; import { ConsoleModule } from './console/console.module';
import { AddonsModule } from './addons/addons.module'; import { AddonsModule } from './addons/addons.module';
import { BackupsModule } from './backups/backups.module'; import { BackupsModule } from './backups/backups.module';
@@ -30,6 +32,7 @@ import { PlansService } from './plans.service';
imports: [ imports: [
AppConfigModule, AppConfigModule,
AuthModule, AuthModule,
SchedulingModule,
forwardRef(() => ConsoleModule), forwardRef(() => ConsoleModule),
forwardRef(() => AddonsModule), forwardRef(() => AddonsModule),
forwardRef(() => BackupsModule), forwardRef(() => BackupsModule),

View File

@@ -19,6 +19,7 @@ import type { GameServer } from '@hexahost/database';
type GameServerStatus = GameServer['status']; type GameServerStatus = GameServer['status'];
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { SchedulingService } from '../scheduling/scheduling.service';
import { findOwnedServer } from './shared/server-ownership.util'; import { findOwnedServer } from './shared/server-ownership.util';
import { ServerStateService } from './server-state.service'; import { ServerStateService } from './server-state.service';
@@ -34,6 +35,7 @@ export class ServersService {
constructor( constructor(
private readonly prisma: PrismaService, private readonly prisma: PrismaService,
private readonly serverState: ServerStateService, private readonly serverState: ServerStateService,
private readonly scheduling: SchedulingService,
@Inject(REDIS_CLIENT) private readonly redis: Redis, @Inject(REDIS_CLIENT) private readonly redis: Redis,
@Inject(SERVER_LIFECYCLE_QUEUE) @Inject(SERVER_LIFECYCLE_QUEUE)
private readonly lifecycleQueue: Queue, private readonly lifecycleQueue: Queue,
@@ -106,6 +108,22 @@ export class ServersService {
this.serverState.assertTransition(server.status, targetStatus); this.serverState.assertTransition(server.status, targetStatus);
const correlationId = randomUUID(); const correlationId = randomUUID();
if (targetStatus === 'STARTING') {
const canStart = await this.scheduling.canStartServer(server);
if (!canStart) {
const queued = await this.scheduling.enqueueStart(
server,
userId,
correlationId,
);
return {
server: this.toResponse(queued.server),
queuePosition: queued.queuePosition,
};
}
}
const updated = await this.transitionServer( const updated = await this.transitionServer(
server, server,
targetStatus, targetStatus,
@@ -153,6 +171,19 @@ export class ServersService {
}; };
} }
async getStartQueue(userId: string, serverId: string) {
return this.scheduling.getStartQueue(userId, serverId);
}
async cancelQueuedStart(
userId: string,
serverId: string,
): Promise<ServerActionResponse> {
await findOwnedServer(this.prisma, userId, serverId);
const updated = await this.scheduling.cancelQueuedStart(userId, serverId);
return { server: this.toResponse(updated) };
}
async stopServer( async stopServer(
userId: string, userId: string,
serverId: string, serverId: string,

View File

@@ -0,0 +1,41 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
describe('phase 6 api contract', () => {
it('documents scheduler, queue and admin node routes', () => {
const files = [
'scheduling/scheduling.service.ts',
'admin/admin-nodes.controller.ts',
'admin/admin-nodes.service.ts',
];
for (const file of files) {
const source = readFileSync(join(process.cwd(), 'src', file), 'utf8');
assert.ok(source.length > 0, `${file} should exist`);
}
const adminSource = readFileSync(
join(process.cwd(), 'src', 'admin/admin-nodes.controller.ts'),
'utf8',
);
assert.ok(adminSource.includes("@Get()"));
assert.ok(adminSource.includes(':id/drain'));
assert.ok(adminSource.includes(':id/maintenance'));
const schedulingSource = readFileSync(
join(process.cwd(), 'src', 'scheduling/scheduling.service.ts'),
'utf8',
);
assert.ok(schedulingSource.includes('enqueueStart'));
assert.ok(schedulingSource.includes('getQueuePosition'));
const serversSource = readFileSync(
join(process.cwd(), 'src', 'servers/servers.controller.ts'),
'utf8',
);
assert.ok(serversSource.includes('start-queue'));
assert.ok(serversSource.includes('cancel-queue'));
});
});

View File

@@ -163,9 +163,11 @@
"PROVISIONING": "Provisionierung", "PROVISIONING": "Provisionierung",
"INSTALLING": "Installation", "INSTALLING": "Installation",
"STOPPED": "Gestoppt", "STOPPED": "Gestoppt",
"QUEUED": "In Warteschlange",
"STARTING": "Startet", "STARTING": "Startet",
"RUNNING": "Läuft", "RUNNING": "Läuft",
"STOPPING": "Stoppt", "STOPPING": "Stoppt",
"UNKNOWN": "Unbekannt",
"ERROR": "Fehler", "ERROR": "Fehler",
"DELETING": "Wird gelöscht", "DELETING": "Wird gelöscht",
"DELETED": "Gelöscht" "DELETED": "Gelöscht"

View File

@@ -163,9 +163,11 @@
"PROVISIONING": "Provisioning", "PROVISIONING": "Provisioning",
"INSTALLING": "Installing", "INSTALLING": "Installing",
"STOPPED": "Stopped", "STOPPED": "Stopped",
"QUEUED": "Queued",
"STARTING": "Starting", "STARTING": "Starting",
"RUNNING": "Running", "RUNNING": "Running",
"STOPPING": "Stopping", "STOPPING": "Stopping",
"UNKNOWN": "Unknown",
"ERROR": "Error", "ERROR": "Error",
"DELETING": "Deleting", "DELETING": "Deleting",
"DELETED": "Deleted" "DELETED": "Deleted"

View File

@@ -15,6 +15,7 @@
"@hexahost/catalog": "workspace:*", "@hexahost/catalog": "workspace:*",
"@hexahost/config": "workspace:*", "@hexahost/config": "workspace:*",
"@hexahost/database": "workspace:*", "@hexahost/database": "workspace:*",
"@hexahost/scheduler": "workspace:*",
"@hexahost/storage": "workspace:*", "@hexahost/storage": "workspace:*",
"bullmq": "^5.34.10", "bullmq": "^5.34.10",
"ioredis": "^5.4.2", "ioredis": "^5.4.2",

View File

@@ -0,0 +1,22 @@
import { prisma } from '@hexahost/database';
export async function releaseServerAllocation(serverId: string): Promise<void> {
const allocation = await prisma.gameServerAllocation.findUnique({
where: { serverId },
});
if (!allocation || allocation.status === 'RELEASED') {
return;
}
await prisma.$transaction([
prisma.gameServerAllocation.update({
where: { serverId },
data: { status: 'RELEASED' },
}),
prisma.gameNode.updateMany({
where: { id: allocation.nodeId, activeServers: { gt: 0 } },
data: { activeServers: { decrement: 1 } },
}),
]);
}

View File

@@ -0,0 +1,105 @@
import { prisma } from '@hexahost/database';
import { HEARTBEAT_STALE_MS } from '@hexahost/scheduler';
import { logger } from '../logger';
import { transitionServerStatus } from '../server-state';
const ACTIVE_SERVER_STATUSES = [
'RUNNING',
'STARTING',
'STOPPING',
] as const;
export async function processNodeHealthTick(): Promise<void> {
const cutoff = new Date(Date.now() - HEARTBEAT_STALE_MS);
const staleNodes = await prisma.gameNode.findMany({
where: {
status: { in: ['ONLINE', 'DRAINING', 'MAINTENANCE'] },
OR: [{ lastHeartbeatAt: null }, { lastHeartbeatAt: { lt: cutoff } }],
},
select: { id: true, name: true },
});
for (const node of staleNodes) {
await markNodeUnreachable(node.id);
logger.warn({ nodeId: node.id, name: node.name }, 'Node marked unreachable');
}
const drainingNodes = await prisma.gameNode.findMany({
where: { status: 'DRAINING' },
include: {
allocations: {
where: { status: 'ACTIVE' },
select: { id: true },
},
},
});
for (const node of drainingNodes) {
if (node.allocations.length === 0 && node.drainRequestedAt) {
await prisma.gameNode.update({
where: { id: node.id },
data: {
status: 'MAINTENANCE',
drainRequestedAt: null,
},
});
logger.info({ nodeId: node.id }, 'Drain complete; node set to maintenance');
}
}
}
async function markNodeUnreachable(nodeId: string): Promise<void> {
await prisma.$transaction(async (tx) => {
const node = await tx.gameNode.findUnique({ where: { id: nodeId } });
if (!node || node.status === 'UNREACHABLE') {
return;
}
await tx.gameNode.update({
where: { id: nodeId },
data: { status: 'UNREACHABLE' },
});
const affectedServers = await tx.gameServer.findMany({
where: {
nodeId,
status: { in: [...ACTIVE_SERVER_STATUSES] },
},
select: { id: true, status: true },
});
for (const server of affectedServers) {
await tx.gameServer.update({
where: { id: server.id },
data: { status: 'UNKNOWN', version: { increment: 1 } },
});
await tx.gameServerStateTransition.create({
data: {
gameServerId: server.id,
fromStatus: server.status,
toStatus: 'UNKNOWN',
reason: 'Node heartbeat lost',
errorCode: 'NODE_UNREACHABLE',
},
});
}
});
}
export async function reconcileNodeOnline(nodeId: string): Promise<void> {
const node = await prisma.gameNode.findUnique({ where: { id: nodeId } });
if (!node || node.status !== 'UNREACHABLE') {
return;
}
await prisma.gameNode.update({
where: { id: nodeId },
data: { status: 'ONLINE' },
});
logger.info({ nodeId }, 'Node reconciled back to online');
}

View File

@@ -1,11 +1,14 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { canStartOnNode } from '@hexahost/scheduler';
import { Job } from 'bullmq'; import { Job } from 'bullmq';
import { z } from 'zod'; import { z } from 'zod';
import { prisma } from '@hexahost/database'; import { prisma } from '@hexahost/database';
import { defaultQueueExpiry } from './start-queue';
import { logger } from '../logger'; import { logger } from '../logger';
import { loadNodeSnapshot } from '../node-snapshots';
import { publishNodeCommand } from '../redis'; import { publishNodeCommand } from '../redis';
import { transitionServerStatus } from '../server-state'; import { transitionServerStatus } from '../server-state';
@@ -64,10 +67,56 @@ export async function processStartServerJob(job: Job): Promise<{ status: string
const { serverId } = lifecycleJobSchema.parse(job.data); const { serverId } = lifecycleJobSchema.parse(job.data);
const server = await loadServerForLifecycle(serverId); const server = await loadServerForLifecycle(serverId);
if (server.status !== 'STOPPED') { if (server.status !== 'STOPPED' && server.status !== 'QUEUED') {
throw new Error(`Cannot start server ${serverId} from status ${server.status}`); throw new Error(`Cannot start server ${serverId} from status ${server.status}`);
} }
const requiredRamMb = server.ramMb;
const nodeSnapshot = await loadNodeSnapshot(server.nodeId!);
if (!nodeSnapshot || !canStartOnNode(nodeSnapshot, requiredRamMb)) {
const correlationId = randomUUID();
await prisma.$transaction(async (tx) => {
await tx.serverStartQueueEntry.upsert({
where: { serverId },
create: {
serverId,
nodeId: server.nodeId!,
status: 'QUEUED',
correlationId,
expiresAt: defaultQueueExpiry(),
},
update: {
status: 'QUEUED',
correlationId,
expiresAt: defaultQueueExpiry(),
requestedAt: new Date(),
},
});
if (server.status !== 'QUEUED') {
await tx.gameServer.update({
where: { id: serverId },
data: { status: 'QUEUED', version: { increment: 1 } },
});
await tx.gameServerStateTransition.create({
data: {
gameServerId: serverId,
fromStatus: server.status,
toStatus: 'QUEUED',
reason: 'Insufficient node capacity; queued for start',
correlationId,
},
});
}
});
logger.info({ serverId, nodeId: server.nodeId }, 'Server start queued');
return { status: 'queued' };
}
const messageId = await publishLifecycleCommand( const messageId = await publishLifecycleCommand(
serverId, serverId,
server.nodeId!, server.nodeId!,
@@ -76,7 +125,7 @@ export async function processStartServerJob(job: Job): Promise<{ status: string
); );
await transitionServerStatus(serverId, 'STARTING', { await transitionServerStatus(serverId, 'STARTING', {
expectedFrom: 'STOPPED', expectedFrom: ['STOPPED', 'QUEUED'],
reason: 'Start requested', reason: 'Start requested',
correlationId: messageId, correlationId: messageId,
}); });

View File

@@ -1,10 +1,13 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { prisma } from '@hexahost/database'; import { prisma } from '@hexahost/database';
import { selectNodeForAllocation } from '@hexahost/scheduler';
import { Job } from 'bullmq'; import { Job } from 'bullmq';
import { z } from 'zod'; import { z } from 'zod';
import { releaseServerAllocation } from '../allocation';
import { logger } from '../logger'; import { logger } from '../logger';
import { loadNodeSnapshots } from '../node-snapshots';
import { publishNodeCommand, waitForNodeResponse } from '../redis'; import { publishNodeCommand, waitForNodeResponse } from '../redis';
import { transitionServerStatus } from '../server-state'; import { transitionServerStatus } from '../server-state';
@@ -65,10 +68,9 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
throw new Error(`Server not found: ${serverId}`); throw new Error(`Server not found: ${serverId}`);
} }
const node = await prisma.gameNode.findFirst({ const nodeSnapshots = await loadNodeSnapshots();
where: { status: 'ONLINE' }, const ramMb = server.plan?.maxRamMb ?? server.ramMb;
orderBy: { createdAt: 'asc' }, const node = selectNodeForAllocation(nodeSnapshots, ramMb);
});
if (!node) { if (!node) {
await transitionServerStatus(serverId, 'ERROR', { await transitionServerStatus(serverId, 'ERROR', {
@@ -80,7 +82,6 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
} }
const hostPort = await findAvailableHostPort(node.id); const hostPort = await findAvailableHostPort(node.id);
const ramMb = server.plan?.maxRamMb ?? server.ramMb;
const dataPath = buildDataPath(serverId); const dataPath = buildDataPath(serverId);
const messageId = randomUUID(); const messageId = randomUUID();
@@ -95,6 +96,11 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
}, },
}); });
await tx.gameNode.update({
where: { id: node.id },
data: { activeServers: { increment: 1 } },
});
await tx.gameServer.update({ await tx.gameServer.update({
where: { id: serverId }, where: { id: serverId },
data: { data: {
@@ -150,6 +156,7 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
} }
if (!response.success) { if (!response.success) {
await releaseServerAllocation(serverId);
await transitionServerStatus(serverId, 'ERROR', { await transitionServerStatus(serverId, 'ERROR', {
reason: 'Provision failed', reason: 'Provision failed',
correlationId: messageId, correlationId: messageId,

View File

@@ -0,0 +1,126 @@
import { randomUUID } from 'node:crypto';
import { prisma } from '@hexahost/database';
import { canStartOnNode } from '@hexahost/scheduler';
import { Queue } from 'bullmq';
import { logger } from '../logger';
import { loadNodeSnapshot } from '../node-snapshots';
import { transitionServerStatus } from '../server-state';
import { QUEUE_SERVER_LIFECYCLE } from '../queues';
import { getRedisPublisher } from '../redis';
const START_QUEUE_EXPIRY_MS = 30 * 60 * 1000;
let lifecycleQueue: Queue | null = null;
function getLifecycleQueue(): Queue {
if (!lifecycleQueue) {
lifecycleQueue = new Queue(QUEUE_SERVER_LIFECYCLE, {
connection: getRedisPublisher(),
});
}
return lifecycleQueue;
}
export async function processStartQueueTick(): Promise<void> {
const entries = await prisma.serverStartQueueEntry.findMany({
where: { status: 'QUEUED' },
orderBy: [{ priority: 'desc' }, { requestedAt: 'asc' }],
include: {
server: {
include: { plan: true },
},
},
take: 25,
});
if (entries.length === 0) {
return;
}
const now = Date.now();
for (const entry of entries) {
if (entry.expiresAt && entry.expiresAt.getTime() < now) {
await expireQueueEntry(entry.serverId);
continue;
}
const requiredRamMb = entry.server.plan?.maxRamMb ?? entry.server.ramMb;
const nodeSnapshot = await loadNodeSnapshot(entry.nodeId);
if (!nodeSnapshot || !canStartOnNode(nodeSnapshot, requiredRamMb)) {
continue;
}
const correlationId = entry.correlationId ?? randomUUID();
await prisma.$transaction(async (tx) => {
const current = await tx.serverStartQueueEntry.findUnique({
where: { id: entry.id },
});
if (!current || current.status !== 'QUEUED') {
return;
}
await tx.serverStartQueueEntry.update({
where: { id: entry.id },
data: { status: 'DISPATCHED', correlationId },
});
});
await getLifecycleQueue().add(
'start-server',
{ serverId: entry.serverId, correlationId },
{ jobId: correlationId },
);
logger.info(
{ serverId: entry.serverId, nodeId: entry.nodeId },
'Dispatched queued server start',
);
}
}
async function expireQueueEntry(serverId: string): Promise<void> {
await prisma.$transaction(async (tx) => {
const entry = await tx.serverStartQueueEntry.findUnique({
where: { serverId },
});
if (!entry || entry.status !== 'QUEUED') {
return;
}
await tx.serverStartQueueEntry.update({
where: { serverId },
data: { status: 'EXPIRED' },
});
const server = await tx.gameServer.findUnique({ where: { id: serverId } });
if (server?.status === 'QUEUED') {
await tx.gameServer.update({
where: { id: serverId },
data: { status: 'STOPPED', version: { increment: 1 } },
});
await tx.gameServerStateTransition.create({
data: {
gameServerId: serverId,
fromStatus: 'QUEUED',
toStatus: 'STOPPED',
reason: 'Start queue entry expired',
},
});
}
});
logger.info({ serverId }, 'Expired queued start request');
}
export function defaultQueueExpiry(): Date {
return new Date(Date.now() + START_QUEUE_EXPIRY_MS);
}

View File

@@ -3,11 +3,13 @@ import { Job, Worker, type ConnectionOptions } from 'bullmq';
import { validateConfig } from '@hexahost/config'; import { validateConfig } from '@hexahost/config';
import { prisma } from '@hexahost/database'; import { prisma } from '@hexahost/database';
import { processNodeHealthTick } from './handlers/node-health';
import { processNotificationJob } from './handlers/notifications'; import { processNotificationJob } from './handlers/notifications';
import { processAddonJob } from './handlers/server-addons'; import { processAddonJob } from './handlers/server-addons';
import { processBackupJob, processScheduledBackupsTick } from './handlers/server-backups'; import { processBackupJob, processScheduledBackupsTick } from './handlers/server-backups';
import { processLifecycleJob } from './handlers/server-lifecycle'; import { processLifecycleJob } from './handlers/server-lifecycle';
import { processProvisionServerJob } from './handlers/server-provisioning'; import { processProvisionServerJob } from './handlers/server-provisioning';
import { processStartQueueTick } from './handlers/start-queue';
import { startHealthServer } from './health'; import { startHealthServer } from './health';
import { logger } from './logger'; import { logger } from './logger';
import { closeRedisConnections } from './redis'; import { closeRedisConnections } from './redis';
@@ -91,6 +93,12 @@ async function bootstrap(): Promise<void> {
void processScheduledBackupsTick().catch((error: Error) => { void processScheduledBackupsTick().catch((error: Error) => {
logger.error({ err: error }, 'Scheduled backup tick failed'); logger.error({ err: error }, 'Scheduled backup tick failed');
}); });
void processStartQueueTick().catch((error: Error) => {
logger.error({ err: error }, 'Start queue tick failed');
});
void processNodeHealthTick().catch((error: Error) => {
logger.error({ err: error }, 'Node health tick failed');
});
}, 60_000); }, 60_000);
const shutdown = async (signal: string): Promise<void> => { const shutdown = async (signal: string): Promise<void> => {

View File

@@ -0,0 +1,59 @@
import { prisma } from '@hexahost/database';
import type { NodeSnapshot } from '@hexahost/scheduler';
function readMemoryField(
payload: Record<string, unknown> | null | undefined,
key: 'memoryTotalBytes' | 'memoryUsedBytes',
): number | null {
const value = payload?.[key];
return typeof value === 'number' ? value : null;
}
export async function loadNodeSnapshots(
nodeIds?: string[],
): Promise<NodeSnapshot[]> {
const nodes = await prisma.gameNode.findMany({
where: nodeIds ? { id: { in: nodeIds } } : undefined,
include: {
allocations: {
where: { status: 'ACTIVE' },
select: { ramMbReserved: true },
},
heartbeats: {
orderBy: { createdAt: 'desc' },
take: 1,
select: { payload: true },
},
},
});
return nodes.map((node) => {
const heartbeatPayload = node.heartbeats[0]?.payload as
| Record<string, unknown>
| undefined;
const reservedRamMb = node.allocations.reduce(
(sum, allocation) => sum + allocation.ramMbReserved,
0,
);
return {
id: node.id,
status: node.status,
maxServers: node.maxServers,
maxRamMb: node.maxRamMb,
platformRamMb: node.platformRamMb,
activeServers: node.activeServers,
lastHeartbeatAt: node.lastHeartbeatAt,
memoryTotalBytes: readMemoryField(heartbeatPayload, 'memoryTotalBytes'),
memoryUsedBytes: readMemoryField(heartbeatPayload, 'memoryUsedBytes'),
reservedRamMb,
activeAllocationCount: node.allocations.length,
};
});
}
export async function loadNodeSnapshot(nodeId: string): Promise<NodeSnapshot | null> {
const snapshots = await loadNodeSnapshots([nodeId]);
return snapshots[0] ?? null;
}

File diff suppressed because one or more lines are too long

View File

@@ -2,51 +2,57 @@
Last updated: 2026-06-26 Last updated: 2026-06-26
## Current phase: Phase 5Software & add-ons ## Current phase: Phase 6Multi-Node and queue
### Phase 5 completed ### Phase 6 completed
| Area | Status | Notes | | Area | Status | Notes |
|------|--------|-------| |------|--------|-------|
| Prisma InstalledAddon | Done | Plugin/mod install tracking | | `@hexahost/scheduler` | Done | Node scoring, RAM/slot capacity, least-loaded placement |
| `@hexahost/catalog` | Done | Modrinth API + compatibility checks | | Prisma start queue + node capacity | Done | `ServerStartQueueEntry`, `maxRamMb`, `platformRamMb`, `QUEUED`/`UNKNOWN` |
| `@hexahost/contracts` phase 5 | Done | catalog, addons schemas | | `@hexahost/contracts` phase 6 | Done | `nodes.ts`, queue schemas, extended server status |
| Go agent v0.5 | Done | addon.install/remove with SHA512 verify | | Worker scheduler integration | Done | Provisioning uses `selectNodeForAllocation` |
| Worker server-addons queue | Done | install-addon, remove-addon jobs | | Start queue | Done | API enqueue + worker tick dispatch |
| API CatalogModule | Done | Modrinth search + versions | | Node health tick | Done | Heartbeat timeout → `UNREACHABLE`, servers → `UNKNOWN` |
| API AddonsModule | Done | Install/remove with audit | | Drain / maintenance | Done | Admin `POST /admin/nodes/:id/drain` and `/maintenance` |
| Web add-ons tab | Done | Search, install, list, remove | | Allocation release | Done | Released on provision failure |
| Docker FABRIC/PURPUR | Done | Extended TYPE mapping for itzg image | | Dev seed | Done | Second node `local-dev-2` for multi-node tests |
### Abnahmekriterium Phase 5 ### Abnahmekriterium Phase 6
- [x] Modrinth catalog search with loader + game version facets - [x] Scheduler distributes servers across eligible nodes (least loaded RAM)
- [x] Compatibility check before install - [x] Resource reservation via `GameServerAllocation` + node RAM budget
- [x] Paper plugin → `plugins/` directory - [x] Start queue when node lacks capacity
- [x] Fabric mod → `mods/` directory - [x] Drain excludes node from new allocations
- [x] SHA512 verification on agent - [x] Maintenance mode blocks scheduling
- [x] Install/remove only when server STOPPED - [x] Node heartbeat loss marks servers `UNKNOWN`
- [ ] Manual E2E (Paper + Fabric server with agent) - [ ] Manual E2E with two agents on different nodes
### Local test ### Local test (two nodes)
1. Create **Paper** server → stop → **Add-ons** → search plugin (e.g. LuckPerms) → install 1. `pnpm db:seed` — note both enrollment tokens
2. Create **Fabric** server → stop → install compatible mod 2. Start agent 1 with `NODE_ID=...001` and agent 2 with `NODE_ID=...002`
3. Start server and verify JAR in container 3. Create several servers and start them — verify different `nodeId` in API responses
4. Saturate one node RAM (`maxRamMb` low) → next server lands on other node
5. `POST /admin/nodes/:id/drain` — new provisions skip drained node
### Known limitations ### Known limitations
- Modpacks/datapacks UI deferred - Offline migration not implemented (manual re-provision only)
- No automatic update/rollback yet (metadata stored for future) - No admin UI for nodes yet (API only)
- CurseForge not integrated - Queue priority uses plan metadata placeholder (`priority=0`)
- Server must be reprovisioned to change software family (existing flow) - CPU overcommit not configurable yet
---
## Phase 5 — Software & add-ons ✅
Modrinth catalog, addon install/remove, SHA512 verify.
--- ---
## Phase 4 — Worlds & backups ✅ ## Phase 4 — Worlds & backups ✅
S3 backups, world upload/download, restore, retention, schedules.
--- ---
## Phase 3 — Console, files, settings ✅ ## Phase 3 — Console, files, settings ✅
@@ -63,6 +69,6 @@ S3 backups, world upload/download, restore, retention, schedules.
## Phase 0 — Monorepo foundation ✅ ## Phase 0 — Monorepo foundation ✅
### Next phase: Phase 6Multi-Node and queue ### Next phase: Phase 7Idle shutdown and free tier
- Scheduler, resource reservation, start queue, drain - Player detection, auto-stop, quotas, credits, usage metering

View File

@@ -180,3 +180,17 @@ export {
type InstallAddonRequest, type InstallAddonRequest,
type InstallAddonResponse, type InstallAddonResponse,
} from './addons'; } from './addons';
export {
gameNodeStatusSchema,
nodeSummarySchema,
nodeListResponseSchema,
startQueueStatusSchema,
startQueueEntrySchema,
startQueueResponseSchema,
type GameNodeStatus,
type NodeSummary,
type NodeListResponse,
type StartQueueEntry,
type StartQueueResponse,
} from './nodes';

View File

@@ -0,0 +1,57 @@
import { z } from 'zod';
export const gameNodeStatusSchema = z.enum([
'ONLINE',
'OFFLINE',
'MAINTENANCE',
'DRAINING',
'UNREACHABLE',
]);
export const nodeSummarySchema = z.object({
id: z.string().uuid(),
name: z.string(),
hostname: z.string(),
status: gameNodeStatusSchema,
maxServers: z.number().int(),
maxRamMb: z.number().int(),
platformRamMb: z.number().int(),
activeServers: z.number().int(),
reservedRamMb: z.number().int(),
availableRamMb: z.number().int(),
lastHeartbeatAt: z.string().datetime().nullable(),
agentVersion: z.string().nullable(),
drainRequestedAt: z.string().datetime().nullable(),
createdAt: z.string().datetime(),
});
export const nodeListResponseSchema = z.object({
nodes: z.array(nodeSummarySchema),
});
export const startQueueStatusSchema = z.enum([
'QUEUED',
'DISPATCHED',
'CANCELLED',
'EXPIRED',
]);
export const startQueueEntrySchema = z.object({
serverId: z.string().uuid(),
nodeId: z.string().uuid(),
status: startQueueStatusSchema,
priority: z.number().int(),
position: z.number().int().nullable(),
requestedAt: z.string().datetime(),
expiresAt: z.string().datetime().nullable(),
});
export const startQueueResponseSchema = z.object({
entry: startQueueEntrySchema.nullable(),
});
export type GameNodeStatus = z.infer<typeof gameNodeStatusSchema>;
export type NodeSummary = z.infer<typeof nodeSummarySchema>;
export type NodeListResponse = z.infer<typeof nodeListResponseSchema>;
export type StartQueueEntry = z.infer<typeof startQueueEntrySchema>;
export type StartQueueResponse = z.infer<typeof startQueueResponseSchema>;

View File

@@ -5,11 +5,13 @@ export const gameServerStatusSchema = z.enum([
'PROVISIONING', 'PROVISIONING',
'INSTALLING', 'INSTALLING',
'STOPPED', 'STOPPED',
'QUEUED',
'STARTING', 'STARTING',
'RUNNING', 'RUNNING',
'STOPPING', 'STOPPING',
'BACKING_UP', 'BACKING_UP',
'RESTORING', 'RESTORING',
'UNKNOWN',
'ERROR', 'ERROR',
'DELETING', 'DELETING',
'DELETED', 'DELETED',
@@ -68,6 +70,7 @@ export const serverListResponseSchema = z.object({
export const serverActionResponseSchema = z.object({ export const serverActionResponseSchema = z.object({
server: serverResponseSchema, server: serverResponseSchema,
jobId: z.string().optional(), jobId: z.string().optional(),
queuePosition: z.number().int().optional(),
}); });
export type GameServerStatus = z.infer<typeof gameServerStatusSchema>; export type GameServerStatus = z.infer<typeof gameServerStatusSchema>;

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,38 @@
-- Phase 6: Multi-node scheduling, start queue, node capacity
CREATE TYPE "StartQueueStatus" AS ENUM ('QUEUED', 'DISPATCHED', 'CANCELLED', 'EXPIRED');
ALTER TYPE "GameServerStatus" ADD VALUE IF NOT EXISTS 'QUEUED';
ALTER TYPE "GameServerStatus" ADD VALUE IF NOT EXISTS 'UNKNOWN';
ALTER TABLE "game_nodes"
ADD COLUMN IF NOT EXISTS "maxRamMb" INTEGER NOT NULL DEFAULT 16384,
ADD COLUMN IF NOT EXISTS "platformRamMb" INTEGER NOT NULL DEFAULT 2048,
ADD COLUMN IF NOT EXISTS "drainRequestedAt" TIMESTAMPTZ(3);
CREATE TABLE "server_start_queue" (
"id" TEXT NOT NULL,
"serverId" TEXT NOT NULL,
"nodeId" TEXT NOT NULL,
"status" "StartQueueStatus" NOT NULL DEFAULT 'QUEUED',
"priority" INTEGER NOT NULL DEFAULT 0,
"correlationId" TEXT,
"requestedAt" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"expiresAt" TIMESTAMPTZ(3),
"createdAt" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updatedAt" TIMESTAMPTZ(3) NOT NULL,
CONSTRAINT "server_start_queue_pkey" PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX "server_start_queue_serverId_key" ON "server_start_queue"("serverId");
CREATE INDEX "server_start_queue_status_priority_requestedAt_idx" ON "server_start_queue"("status", "priority", "requestedAt");
CREATE INDEX "server_start_queue_nodeId_idx" ON "server_start_queue"("nodeId");
ALTER TABLE "server_start_queue"
ADD CONSTRAINT "server_start_queue_serverId_fkey"
FOREIGN KEY ("serverId") REFERENCES "game_servers"("id") ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE "server_start_queue"
ADD CONSTRAINT "server_start_queue_nodeId_fkey"
FOREIGN KEY ("nodeId") REFERENCES "game_nodes"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

View File

@@ -34,11 +34,13 @@ enum GameServerStatus {
PROVISIONING PROVISIONING
INSTALLING INSTALLING
STOPPED STOPPED
QUEUED
STARTING STARTING
RUNNING RUNNING
STOPPING STOPPING
BACKING_UP BACKING_UP
RESTORING RESTORING
UNKNOWN
ERROR ERROR
DELETING DELETING
DELETED DELETED
@@ -57,6 +59,13 @@ enum AllocationStatus {
RELEASED RELEASED
} }
enum StartQueueStatus {
QUEUED
DISPATCHED
CANCELLED
EXPIRED
}
enum JobStatus { enum JobStatus {
PENDING PENDING
RUNNING RUNNING
@@ -282,6 +291,7 @@ model GameServer {
backupSchedule BackupSchedule? backupSchedule BackupSchedule?
worldUploads WorldUpload[] worldUploads WorldUpload[]
installedAddons InstalledAddon[] installedAddons InstalledAddon[]
startQueueEntry ServerStartQueueEntry?
@@index([userId]) @@index([userId])
@@index([nodeId]) @@index([nodeId])
@@ -333,11 +343,14 @@ model GameNode {
hostname String hostname String
status GameNodeStatus @default(OFFLINE) status GameNodeStatus @default(OFFLINE)
maxServers Int @default(10) maxServers Int @default(10)
maxRamMb Int @default(16384)
platformRamMb Int @default(2048)
activeServers Int @default(0) activeServers Int @default(0)
lastHeartbeatAt DateTime? @db.Timestamptz(3) lastHeartbeatAt DateTime? @db.Timestamptz(3)
agentVersion String? agentVersion String?
enrollmentTokenHash String? enrollmentTokenHash String?
enrolledAt DateTime? @db.Timestamptz(3) enrolledAt DateTime? @db.Timestamptz(3)
drainRequestedAt DateTime? @db.Timestamptz(3)
metadata Json? metadata Json?
createdAt DateTime @default(now()) @db.Timestamptz(3) createdAt DateTime @default(now()) @db.Timestamptz(3)
updatedAt DateTime @updatedAt @db.Timestamptz(3) updatedAt DateTime @updatedAt @db.Timestamptz(3)
@@ -345,6 +358,7 @@ model GameNode {
gameServers GameServer[] gameServers GameServer[]
heartbeats GameNodeHeartbeat[] heartbeats GameNodeHeartbeat[]
allocations GameServerAllocation[] allocations GameServerAllocation[]
startQueueEntries ServerStartQueueEntry[]
@@map("game_nodes") @@map("game_nodes")
} }
@@ -362,6 +376,26 @@ model GameNodeHeartbeat {
@@map("game_node_heartbeats") @@map("game_node_heartbeats")
} }
model ServerStartQueueEntry {
id String @id @default(uuid())
serverId String @unique
nodeId String
status StartQueueStatus @default(QUEUED)
priority Int @default(0)
correlationId String?
requestedAt DateTime @default(now()) @db.Timestamptz(3)
expiresAt DateTime? @db.Timestamptz(3)
createdAt DateTime @default(now()) @db.Timestamptz(3)
updatedAt DateTime @updatedAt @db.Timestamptz(3)
server GameServer @relation(fields: [serverId], references: [id], onDelete: Cascade)
node GameNode @relation(fields: [nodeId], references: [id])
@@index([status, priority, requestedAt])
@@index([nodeId])
@@map("server_start_queue")
}
model Plan { model Plan {
id String @id @default(uuid()) id String @id @default(uuid())
name String @unique name String @unique

View File

@@ -5,10 +5,15 @@ import { hashToken } from '@hexahost/auth';
/** Stable dev node ID — must match NODE_ID in .env for local agent */ /** Stable dev node ID — must match NODE_ID in .env for local agent */
export const DEV_NODE_ID = '00000000-0000-4000-8000-000000000001'; export const DEV_NODE_ID = '00000000-0000-4000-8000-000000000001';
export const DEV_NODE_ID_2 = '00000000-0000-4000-8000-000000000002';
/** Shown once in seed output; hash stored in DB */ /** Shown once in seed output; hash stored in DB */
export const DEV_NODE_ENROLLMENT_TOKEN = export const DEV_NODE_ENROLLMENT_TOKEN =
'local-dev-enrollment-token-change-me-32chars'; 'local-dev-enrollment-token-change-me-32chars';
export const DEV_NODE_2_ENROLLMENT_TOKEN =
'local-dev-node2-enrollment-token-32ch';
async function main(): Promise<void> { async function main(): Promise<void> {
const prisma = new PrismaClient(); const prisma = new PrismaClient();
@@ -38,19 +43,46 @@ async function main(): Promise<void> {
hostname: 'localhost', hostname: 'localhost',
status: 'OFFLINE', status: 'OFFLINE',
maxServers: 10, maxServers: 10,
maxRamMb: 16384,
platformRamMb: 2048,
enrollmentTokenHash: hashToken(DEV_NODE_ENROLLMENT_TOKEN), enrollmentTokenHash: hashToken(DEV_NODE_ENROLLMENT_TOKEN),
}, },
update: { update: {
name: 'local-dev', name: 'local-dev',
hostname: 'localhost', hostname: 'localhost',
maxRamMb: 16384,
platformRamMb: 2048,
enrollmentTokenHash: hashToken(DEV_NODE_ENROLLMENT_TOKEN), enrollmentTokenHash: hashToken(DEV_NODE_ENROLLMENT_TOKEN),
}, },
}); });
const devNode2 = await prisma.gameNode.upsert({
where: { id: DEV_NODE_ID_2 },
create: {
id: DEV_NODE_ID_2,
name: 'local-dev-2',
hostname: 'localhost',
status: 'OFFLINE',
maxServers: 10,
maxRamMb: 16384,
platformRamMb: 2048,
enrollmentTokenHash: hashToken(DEV_NODE_2_ENROLLMENT_TOKEN),
},
update: {
name: 'local-dev-2',
hostname: 'localhost',
maxRamMb: 16384,
platformRamMb: 2048,
enrollmentTokenHash: hashToken(DEV_NODE_2_ENROLLMENT_TOKEN),
},
});
console.log('Seed complete:'); console.log('Seed complete:');
console.log(` Plan Free: ${freePlan.id}`); console.log(` Plan Free: ${freePlan.id}`);
console.log(` Dev node: ${devNode.id} (${devNode.name})`); console.log(` Dev node: ${devNode.id} (${devNode.name})`);
console.log(` Dev node 2: ${devNode2.id} (${devNode2.name})`);
console.log(` Enrollment token (set NODE_TOKEN): ${DEV_NODE_ENROLLMENT_TOKEN}`); console.log(` Enrollment token (set NODE_TOKEN): ${DEV_NODE_ENROLLMENT_TOKEN}`);
console.log(` Node 2 enrollment token: ${DEV_NODE_2_ENROLLMENT_TOKEN}`);
} finally { } finally {
await prisma.$disconnect(); await prisma.$disconnect();
} }

View File

@@ -19,6 +19,7 @@ export type {
InstalledAddon, InstalledAddon,
GameNode, GameNode,
GameNodeHeartbeat, GameNodeHeartbeat,
ServerStartQueueEntry,
Plan, Plan,
FeatureFlag, FeatureFlag,
SystemSetting, SystemSetting,

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,18 @@
{
"name": "@hexahost/scheduler",
"version": "0.0.0",
"private": true,
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"scripts": {
"build": "tsc",
"typecheck": "tsc --noEmit",
"test": "vitest run"
},
"devDependencies": {
"@hexahost/typescript-config": "workspace:*",
"@types/node": "^22.10.2",
"typescript": "^5.7.3",
"vitest": "^3.0.8"
}
}

View File

@@ -0,0 +1,98 @@
import type { NodeCapacity, NodeSnapshot } from './types';
export const HEARTBEAT_STALE_MS = 90_000;
/** Nodes that accept new server allocations (provisioning). */
export const ALLOCATION_ELIGIBLE_STATUSES = ['ONLINE'] as const;
/** Nodes that can start already-provisioned servers. */
export const START_ELIGIBLE_STATUSES = ['ONLINE', 'DRAINING'] as const;
export function isHeartbeatFresh(
lastHeartbeatAt: Date | null,
now = Date.now(),
): boolean {
if (!lastHeartbeatAt) {
return false;
}
return now - lastHeartbeatAt.getTime() <= HEARTBEAT_STALE_MS;
}
export function effectiveMaxRamMb(node: NodeSnapshot): number {
return Math.max(node.maxRamMb - node.platformRamMb, 0);
}
export function computeAvailableRamMb(node: NodeSnapshot): number {
const budget = effectiveMaxRamMb(node);
if (
node.memoryTotalBytes != null &&
node.memoryUsedBytes != null &&
node.memoryTotalBytes > 0
) {
const hostFreeMb = Math.floor(
(node.memoryTotalBytes - node.memoryUsedBytes) / (1024 * 1024),
);
return Math.max(0, Math.min(budget - node.reservedRamMb, hostFreeMb));
}
return Math.max(0, budget - node.reservedRamMb);
}
export function computeAvailableSlots(node: NodeSnapshot): number {
const byMax = Math.max(0, node.maxServers - node.activeAllocationCount);
const byActive = Math.max(0, node.maxServers - node.activeServers);
return Math.min(byMax, byActive);
}
export function canAllocateOnNode(node: NodeSnapshot, requiredRamMb: number): boolean {
if (!ALLOCATION_ELIGIBLE_STATUSES.includes(node.status as (typeof ALLOCATION_ELIGIBLE_STATUSES)[number])) {
return false;
}
if (!isHeartbeatFresh(node.lastHeartbeatAt)) {
return false;
}
const capacity = evaluateNodeCapacity(node, requiredRamMb);
return capacity.schedulable;
}
export function canStartOnNode(node: NodeSnapshot, requiredRamMb: number): boolean {
if (!START_ELIGIBLE_STATUSES.includes(node.status as (typeof START_ELIGIBLE_STATUSES)[number])) {
return false;
}
if (!isHeartbeatFresh(node.lastHeartbeatAt)) {
return false;
}
const capacity = evaluateNodeCapacity(node, requiredRamMb);
return capacity.schedulable;
}
export function evaluateNodeCapacity(
node: NodeSnapshot,
requiredRamMb: number,
): NodeCapacity {
const availableRamMb = computeAvailableRamMb(node);
const availableSlots = computeAvailableSlots(node);
const schedulable =
requiredRamMb > 0 &&
availableRamMb >= requiredRamMb &&
availableSlots > 0;
const ramHeadroom = availableRamMb - requiredRamMb;
const slotHeadroom = availableSlots - 1;
const score = schedulable
? ramHeadroom * 10 + slotHeadroom * 100 - node.reservedRamMb
: -1;
return {
schedulable,
availableRamMb,
availableSlots,
score,
};
}

View File

@@ -0,0 +1,19 @@
export type { GameNodeStatus, NodeCapacity, NodeSnapshot } from './types';
export {
ALLOCATION_ELIGIBLE_STATUSES,
START_ELIGIBLE_STATUSES,
HEARTBEAT_STALE_MS,
isHeartbeatFresh,
effectiveMaxRamMb,
computeAvailableRamMb,
computeAvailableSlots,
canAllocateOnNode,
canStartOnNode,
evaluateNodeCapacity,
} from './capacity';
export {
selectNodeForAllocation,
findNodeSnapshot,
} from './select-node';

View File

@@ -0,0 +1,56 @@
import { describe, expect, it } from 'vitest';
import {
canAllocateOnNode,
canStartOnNode,
selectNodeForAllocation,
} from './index';
import type { NodeSnapshot } from './types';
function node(partial: Partial<NodeSnapshot> & Pick<NodeSnapshot, 'id'>): NodeSnapshot {
return {
status: 'ONLINE',
maxServers: 10,
maxRamMb: 16384,
platformRamMb: 2048,
activeServers: 0,
lastHeartbeatAt: new Date(),
reservedRamMb: 0,
activeAllocationCount: 0,
...partial,
};
}
describe('scheduler capacity', () => {
it('rejects maintenance and draining nodes for new allocations', () => {
const maintenance = node({ id: 'a', status: 'MAINTENANCE' });
const draining = node({ id: 'b', status: 'DRAINING' });
expect(canAllocateOnNode(maintenance, 1024)).toBe(false);
expect(canAllocateOnNode(draining, 1024)).toBe(false);
expect(canStartOnNode(draining, 1024)).toBe(true);
});
it('picks the least loaded node for placement', () => {
const lightlyLoaded = node({ id: 'light', reservedRamMb: 1024 });
const heavilyLoaded = node({ id: 'heavy', reservedRamMb: 8192 });
const selected = selectNodeForAllocation(
[heavilyLoaded, lightlyLoaded],
2048,
);
expect(selected?.id).toBe('light');
});
it('returns null when no node has enough RAM', () => {
const saturated = node({
id: 'full',
reservedRamMb: 14000,
maxRamMb: 16384,
platformRamMb: 2048,
});
expect(selectNodeForAllocation([saturated], 4096)).toBeNull();
});
});

View File

@@ -0,0 +1,30 @@
import { canAllocateOnNode, evaluateNodeCapacity } from './capacity';
import type { NodeSnapshot } from './types';
export function selectNodeForAllocation(
nodes: NodeSnapshot[],
requiredRamMb: number,
): NodeSnapshot | null {
const candidates = nodes
.filter((node) => canAllocateOnNode(node, requiredRamMb))
.map((node) => ({
node,
capacity: evaluateNodeCapacity(node, requiredRamMb),
}))
.sort((left, right) => {
if (right.capacity.score !== left.capacity.score) {
return right.capacity.score - left.capacity.score;
}
return left.node.reservedRamMb - right.node.reservedRamMb;
});
return candidates[0]?.node ?? null;
}
export function findNodeSnapshot(
nodes: NodeSnapshot[],
nodeId: string,
): NodeSnapshot | null {
return nodes.find((node) => node.id === nodeId) ?? null;
}

View File

@@ -0,0 +1,27 @@
export type GameNodeStatus =
| 'ONLINE'
| 'OFFLINE'
| 'MAINTENANCE'
| 'DRAINING'
| 'UNREACHABLE';
export interface NodeSnapshot {
id: string;
status: GameNodeStatus;
maxServers: number;
maxRamMb: number;
platformRamMb: number;
activeServers: number;
lastHeartbeatAt: Date | null;
memoryTotalBytes?: number | null;
memoryUsedBytes?: number | null;
reservedRamMb: number;
activeAllocationCount: number;
}
export interface NodeCapacity {
schedulable: boolean;
availableRamMb: number;
availableSlots: number;
score: number;
}

View File

@@ -0,0 +1,9 @@
{
"extends": "@hexahost/typescript-config/base.json",
"compilerOptions": {
"outDir": "./dist",
"rootDir": "./src"
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}

21
pnpm-lock.yaml generated
View File

@@ -41,6 +41,9 @@ importers:
'@hexahost/database': '@hexahost/database':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/database version: link:../../packages/database
'@hexahost/scheduler':
specifier: workspace:*
version: link:../../packages/scheduler
'@hexahost/storage': '@hexahost/storage':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/storage version: link:../../packages/storage
@@ -184,6 +187,9 @@ importers:
'@hexahost/database': '@hexahost/database':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/database version: link:../../packages/database
'@hexahost/scheduler':
specifier: workspace:*
version: link:../../packages/scheduler
'@hexahost/storage': '@hexahost/storage':
specifier: workspace:* specifier: workspace:*
version: link:../../packages/storage version: link:../../packages/storage
@@ -315,6 +321,21 @@ importers:
packages/eslint-config: {} packages/eslint-config: {}
packages/scheduler:
devDependencies:
'@hexahost/typescript-config':
specifier: workspace:*
version: link:../typescript-config
'@types/node':
specifier: ^22.10.2
version: 22.20.0
typescript:
specifier: ^5.7.3
version: 5.9.3
vitest:
specifier: ^3.0.8
version: 3.2.6(@types/node@22.20.0)(jiti@2.7.0)(terser@5.48.0)(tsx@4.22.4)
packages/storage: packages/storage:
dependencies: dependencies:
'@aws-sdk/client-s3': '@aws-sdk/client-s3':