Phase4
Some checks failed
CI / Node — lint, typecheck, test, build (push) Failing after 9s
CI / Go — node-agent tests (push) Failing after 9s
CI / Go — edge-gateway build (push) Successful in 13s

This commit is contained in:
smueller
2026-06-26 12:32:27 +02:00
parent 9b061c3ee7
commit 4262464cd5
101 changed files with 4024 additions and 74 deletions

View File

@@ -19,6 +19,7 @@
"@hexahost/config": "workspace:*",
"@hexahost/contracts": "workspace:*",
"@hexahost/database": "workspace:*",
"@hexahost/storage": "workspace:*",
"@nestjs/common": "^11.0.7",
"@nestjs/core": "^11.0.7",
"@nestjs/platform-fastify": "^11.0.7",

View File

@@ -320,7 +320,14 @@ export class NodesService {
normalized === 'files.list' ||
normalized === 'files.read' ||
normalized === 'files.write' ||
normalized === 'command'
normalized === 'command' ||
normalized === 'backup.prepare' ||
normalized === 'backup.release' ||
normalized === 'world.validate' ||
normalized === 'world.archive' ||
normalized === 'world.replace' ||
normalized === 'storage.upload' ||
normalized === 'storage.download'
);
}

View File

@@ -0,0 +1,102 @@
import {
Body,
Controller,
Delete,
Get,
Param,
ParseUUIDPipe,
Patch,
Post,
UseGuards,
} from '@nestjs/common';
import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger';
import {
createBackupRequestSchema,
restoreBackupRequestSchema,
updateBackupScheduleSchema,
} from '@hexahost/contracts';
import { CurrentUser } from '../../auth/decorators/current-user.decorator';
import { SessionGuard } from '../../auth/guards/session.guard';
import { BackupsService } from './backups.service';
@ApiTags('servers')
@Controller('servers/:serverId/backups')
@UseGuards(SessionGuard)
export class BackupsController {
constructor(private readonly backupsService: BackupsService) {}
@Get()
@ApiOperation({ summary: 'List server backups' })
list(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
) {
return this.backupsService.listBackups(user.id, serverId);
}
@Post()
@ApiOperation({ summary: 'Create a manual backup' })
create(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Body() body: unknown,
) {
const input = createBackupRequestSchema.parse(body ?? {});
return this.backupsService.createBackup(user.id, serverId, input);
}
@Get('schedule')
@ApiOperation({ summary: 'Get backup schedule' })
getSchedule(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
) {
return this.backupsService.getSchedule(user.id, serverId);
}
@Patch('schedule')
@ApiOperation({ summary: 'Update backup schedule' })
updateSchedule(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Body() body: unknown,
) {
const input = updateBackupScheduleSchema.parse(body);
return this.backupsService.updateSchedule(user.id, serverId, input);
}
@Get(':backupId/download-url')
@ApiOperation({ summary: 'Get a presigned download URL for a backup' })
downloadUrl(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Param('backupId', ParseUUIDPipe) backupId: string,
) {
return this.backupsService.getDownloadUrl(user.id, serverId, backupId);
}
@Post(':backupId/restore')
@ApiOperation({ summary: 'Restore a backup' })
restore(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Param('backupId', ParseUUIDPipe) backupId: string,
@Body() body: unknown,
) {
restoreBackupRequestSchema.parse(body);
return this.backupsService.restoreBackup(user.id, serverId, backupId);
}
@Delete(':backupId')
@ApiOperation({ summary: 'Delete a backup' })
deleteBackup(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Param('backupId', ParseUUIDPipe) backupId: string,
) {
return this.backupsService.deleteBackup(user.id, serverId, backupId);
}
}

View File

@@ -0,0 +1,34 @@
import { Module } from '@nestjs/common';
import { Queue } from 'bullmq';
import { getConfig } from '@hexahost/config';
import { AuthModule } from '../../auth/auth.module';
import { PrismaModule } from '../../prisma/prisma.module';
import { BackupsController } from './backups.controller';
import { BackupsService, SERVER_BACKUPS_QUEUE } from './backups.service';
import { ServerStateService } from '../server-state.service';
@Module({
imports: [AuthModule, PrismaModule],
controllers: [BackupsController],
providers: [
BackupsService,
ServerStateService,
{
provide: SERVER_BACKUPS_QUEUE,
useFactory: () => {
const config = getConfig();
return new Queue('server-backups', {
connection: {
url: config.REDIS_URL,
maxRetriesPerRequest: null,
},
});
},
},
],
exports: [BackupsService, SERVER_BACKUPS_QUEUE],
})
export class BackupsModule {}

View File

@@ -0,0 +1,341 @@
import {
BadRequestException,
Inject,
Injectable,
} from '@nestjs/common';
import { Queue } from 'bullmq';
import type { ServerBackup } from '@hexahost/database';
import type {
BackupActionResponse,
BackupDownloadUrlResponse,
BackupListResponse,
BackupResponse,
BackupRestoreResponse,
BackupSchedule,
CreateBackupRequest,
UpdateBackupSchedule,
} from '@hexahost/contracts';
import {
createPresignedGetUrl,
createS3Client,
deleteStoredObject,
loadStorageConfig,
} from '@hexahost/storage';
import { AuditService } from '../../auth/audit.service';
import { PrismaService } from '../../prisma/prisma.service';
import { findOwnedServer } from '../shared/server-ownership.util';
import { ServerStateService } from '../server-state.service';
export const SERVER_BACKUPS_QUEUE = Symbol('SERVER_BACKUPS_QUEUE');
@Injectable()
export class BackupsService {
constructor(
private readonly prisma: PrismaService,
private readonly serverState: ServerStateService,
private readonly auditService: AuditService,
@Inject(SERVER_BACKUPS_QUEUE) private readonly backupsQueue: Queue,
) {}
async listBackups(userId: string, serverId: string): Promise<BackupListResponse> {
await findOwnedServer(this.prisma, userId, serverId);
const backups = await this.prisma.serverBackup.findMany({
where: {
serverId,
status: { not: 'DELETED' },
},
orderBy: { createdAt: 'desc' },
});
return { backups: backups.map((backup) => this.toResponse(backup)) };
}
async createBackup(
userId: string,
serverId: string,
input: CreateBackupRequest,
): Promise<BackupActionResponse> {
const server = await findOwnedServer(this.prisma, userId, serverId);
this.serverState.assertNotInProgress(server.status);
if (!server.nodeId) {
throw new BadRequestException('Server is not assigned to a node');
}
if (server.status !== 'RUNNING' && server.status !== 'STOPPED') {
throw new BadRequestException(
`Cannot create backup while server is ${server.status}`,
);
}
const previousStatus = server.status;
const backup = await this.prisma.serverBackup.create({
data: {
serverId,
type: 'MANUAL',
status: 'PENDING',
label: input.label ?? 'Manual backup',
},
});
await this.prisma.gameServer.update({
where: { id: serverId },
data: { status: 'BACKING_UP' },
});
await this.prisma.gameServerStateTransition.create({
data: {
gameServerId: serverId,
fromStatus: previousStatus,
toStatus: 'BACKING_UP',
reason: 'Manual backup requested',
actorId: userId,
},
});
const job = await this.backupsQueue.add('create-backup', {
backupId: backup.id,
serverId,
previousStatus,
});
await this.auditService.record({
action: 'server.backup.create',
userId,
entityType: 'game_server',
entityId: serverId,
metadata: { backupId: backup.id },
});
return {
backup: this.toResponse(backup),
jobId: job.id?.toString(),
};
}
async getDownloadUrl(
userId: string,
serverId: string,
backupId: string,
): Promise<BackupDownloadUrlResponse> {
await findOwnedServer(this.prisma, userId, serverId);
const backup = await this.prisma.serverBackup.findFirstOrThrow({
where: { id: backupId, serverId, status: 'AVAILABLE' },
});
if (!backup.s3Key) {
throw new BadRequestException('Backup is not stored yet');
}
const config = loadStorageConfig();
const client = createS3Client(config);
const expiresAt = new Date(Date.now() + 15 * 60 * 1000);
const downloadUrl = await createPresignedGetUrl(
client,
config.S3_BUCKET,
backup.s3Key,
15 * 60,
);
return {
downloadUrl,
expiresAt: expiresAt.toISOString(),
};
}
async deleteBackup(
userId: string,
serverId: string,
backupId: string,
): Promise<BackupResponse> {
await findOwnedServer(this.prisma, userId, serverId);
const backup = await this.prisma.serverBackup.findFirstOrThrow({
where: { id: backupId, serverId },
});
if (backup.s3Key) {
const config = loadStorageConfig();
const client = createS3Client(config);
await deleteStoredObject(client, config.S3_BUCKET, backup.s3Key);
}
const updated = await this.prisma.serverBackup.update({
where: { id: backupId },
data: { status: 'DELETED' },
});
await this.auditService.record({
action: 'server.backup.delete',
userId,
entityType: 'game_server',
entityId: serverId,
metadata: { backupId },
});
return this.toResponse(updated);
}
async restoreBackup(
userId: string,
serverId: string,
backupId: string,
): Promise<BackupRestoreResponse> {
const server = await findOwnedServer(this.prisma, userId, serverId);
this.serverState.assertNotInProgress(server.status);
if (!server.nodeId) {
throw new BadRequestException('Server is not assigned to a node');
}
if (server.status === 'RUNNING') {
throw new BadRequestException('Server must be stopped before restore');
}
if (server.status !== 'STOPPED') {
throw new BadRequestException(
`Cannot restore while server is ${server.status}`,
);
}
const backup = await this.prisma.serverBackup.findFirstOrThrow({
where: { id: backupId, serverId, status: 'AVAILABLE' },
});
const restore = await this.prisma.backupRestore.create({
data: {
serverId,
backupId: backup.id,
status: 'PENDING',
},
});
await this.prisma.gameServer.update({
where: { id: serverId },
data: { status: 'RESTORING' },
});
await this.prisma.gameServerStateTransition.create({
data: {
gameServerId: serverId,
fromStatus: 'STOPPED',
toStatus: 'RESTORING',
reason: 'Restore requested',
actorId: userId,
},
});
const job = await this.backupsQueue.add('restore-backup', {
restoreId: restore.id,
serverId,
backupId: backup.id,
});
await this.auditService.record({
action: 'server.backup.restore',
userId,
entityType: 'game_server',
entityId: serverId,
metadata: { backupId, restoreId: restore.id },
});
return {
restore: {
id: restore.id,
serverId,
backupId: backup.id,
safetyBackupId: restore.safetyBackupId,
status: restore.status,
failureReason: restore.failureReason,
createdAt: restore.createdAt.toISOString(),
completedAt: restore.completedAt?.toISOString() ?? null,
},
jobId: job.id?.toString(),
};
}
async getSchedule(userId: string, serverId: string): Promise<BackupSchedule> {
await findOwnedServer(this.prisma, userId, serverId);
const schedule = await this.prisma.backupSchedule.upsert({
where: { serverId },
create: {
serverId,
enabled: false,
intervalHours: 24,
retentionCount: 5,
nextRunAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
},
update: {},
});
return this.toScheduleResponse(schedule);
}
async updateSchedule(
userId: string,
serverId: string,
input: UpdateBackupSchedule,
): Promise<BackupSchedule> {
await findOwnedServer(this.prisma, userId, serverId);
const schedule = await this.prisma.backupSchedule.upsert({
where: { serverId },
create: {
serverId,
enabled: input.enabled ?? false,
intervalHours: input.intervalHours ?? 24,
retentionCount: input.retentionCount ?? 5,
nextRunAt: new Date(Date.now() + (input.intervalHours ?? 24) * 60 * 60 * 1000),
},
update: {
enabled: input.enabled,
intervalHours: input.intervalHours,
retentionCount: input.retentionCount,
nextRunAt:
input.enabled === true
? new Date(Date.now() + (input.intervalHours ?? 24) * 60 * 60 * 1000)
: undefined,
},
});
return this.toScheduleResponse(schedule);
}
private toResponse(backup: ServerBackup): BackupResponse {
return {
id: backup.id,
serverId: backup.serverId,
type: backup.type,
status: backup.status,
s3Key: backup.s3Key,
sizeBytes: backup.sizeBytes?.toString() ?? null,
sha256: backup.sha256,
label: backup.label,
failureReason: backup.failureReason,
createdAt: backup.createdAt.toISOString(),
completedAt: backup.completedAt?.toISOString() ?? null,
expiresAt: backup.expiresAt?.toISOString() ?? null,
};
}
private toScheduleResponse(schedule: {
enabled: boolean;
intervalHours: number;
retentionCount: number;
nextRunAt: Date | null;
lastRunAt: Date | null;
}): BackupSchedule {
return {
enabled: schedule.enabled,
intervalHours: schedule.intervalHours,
retentionCount: schedule.retentionCount,
nextRunAt: schedule.nextRunAt?.toISOString() ?? null,
lastRunAt: schedule.lastRunAt?.toISOString() ?? null,
};
}
}

View File

@@ -7,10 +7,12 @@ const ALLOWED_TRANSITIONS: Record<GameServerStatus, GameServerStatus[]> = {
DRAFT: ['PROVISIONING', 'DELETING'],
PROVISIONING: ['INSTALLING', 'ERROR', 'DELETING'],
INSTALLING: ['STOPPED', 'ERROR', 'DELETING'],
STOPPED: ['STARTING', 'PROVISIONING', 'DELETING'],
STOPPED: ['STARTING', 'PROVISIONING', 'BACKING_UP', 'RESTORING', 'DELETING'],
STARTING: ['RUNNING', 'STOPPED', 'ERROR', 'DELETING'],
RUNNING: ['STOPPING', 'ERROR', 'DELETING'],
RUNNING: ['STOPPING', 'BACKING_UP', 'ERROR', 'DELETING'],
STOPPING: ['STOPPED', 'ERROR', 'DELETING'],
BACKING_UP: ['RUNNING', 'STOPPED', 'ERROR'],
RESTORING: ['STOPPED', 'ERROR'],
ERROR: ['STOPPED', 'STARTING', 'PROVISIONING', 'DELETING'],
DELETING: ['DELETED'],
DELETED: [],
@@ -21,6 +23,8 @@ const IN_PROGRESS_STATUSES: GameServerStatus[] = [
'INSTALLING',
'STARTING',
'STOPPING',
'BACKING_UP',
'RESTORING',
'DELETING',
];

View File

@@ -9,7 +9,9 @@ import { NodeBridgeModule } from '../nodes/node-bridge.module';
import { AppConfigModule } from '../config/app-config.module';
import { ConsoleModule } from './console/console.module';
import { BackupsModule } from './backups/backups.module';
import { FilesModule } from './files/files.module';
import { WorldsModule } from './worlds/worlds.module';
import { PlayersModule } from './players/players.module';
import { PropertiesModule } from './properties/properties.module';
import { ServerStateService } from './server-state.service';
@@ -28,6 +30,8 @@ import { PlansService } from './plans.service';
AppConfigModule,
AuthModule,
forwardRef(() => ConsoleModule),
forwardRef(() => BackupsModule),
forwardRef(() => WorldsModule),
forwardRef(() => FilesModule),
forwardRef(() => PropertiesModule),
forwardRef(() => PlayersModule),

View File

@@ -314,6 +314,7 @@ export class ServersService {
hostPort: server.hostPort,
containerId: server.containerId,
dataPath: server.dataPath,
activeWorldName: server.activeWorldName,
ramMb: server.ramMb,
createdAt: server.createdAt.toISOString(),
updatedAt: server.updatedAt.toISOString(),

View File

@@ -0,0 +1,63 @@
import {
Body,
Controller,
Get,
Param,
ParseUUIDPipe,
Post,
UseGuards,
} from '@nestjs/common';
import { ApiOperation, ApiTags } from '@nestjs/swagger';
import { initiateWorldUploadSchema } from '@hexahost/contracts';
import { CurrentUser } from '../../auth/decorators/current-user.decorator';
import { SessionGuard } from '../../auth/guards/session.guard';
import { WorldsService } from './worlds.service';
@ApiTags('servers')
@Controller('servers/:serverId/worlds')
@UseGuards(SessionGuard)
export class WorldsController {
constructor(private readonly worldsService: WorldsService) {}
@Get()
@ApiOperation({ summary: 'Get world overview' })
info(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
) {
return this.worldsService.getWorldInfo(user.id, serverId);
}
@Post('uploads')
@ApiOperation({ summary: 'Initiate a world upload' })
initiateUpload(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Body() body: unknown,
) {
const input = initiateWorldUploadSchema.parse(body);
return this.worldsService.initiateUpload(user.id, serverId, input);
}
@Post('uploads/:uploadId/complete')
@ApiOperation({ summary: 'Complete a world upload and apply it' })
completeUpload(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
@Param('uploadId', ParseUUIDPipe) uploadId: string,
) {
return this.worldsService.completeUpload(user.id, serverId, uploadId);
}
@Post('download')
@ApiOperation({ summary: 'Create a backup for world download' })
download(
@CurrentUser() user: { id: string },
@Param('serverId', ParseUUIDPipe) serverId: string,
) {
return this.worldsService.requestWorldDownload(user.id, serverId);
}
}

View File

@@ -0,0 +1,17 @@
import { Module } from '@nestjs/common';
import { AuthModule } from '../../auth/auth.module';
import { NodeBridgeModule } from '../../nodes/node-bridge.module';
import { PrismaModule } from '../../prisma/prisma.module';
import { BackupsModule } from '../backups/backups.module';
import { ServerStateService } from '../server-state.service';
import { WorldsController } from './worlds.controller';
import { WorldsService } from './worlds.service';
@Module({
imports: [AuthModule, PrismaModule, NodeBridgeModule, BackupsModule],
controllers: [WorldsController],
providers: [WorldsService, ServerStateService],
})
export class WorldsModule {}

View File

@@ -0,0 +1,250 @@
import {
BadRequestException,
Inject,
Injectable,
ServiceUnavailableException,
} from '@nestjs/common';
import { Queue } from 'bullmq';
import { randomUUID } from 'node:crypto';
import type {
InitiateWorldUpload,
WorldDownloadUrlResponse,
WorldInfoResponse,
WorldUploadInitResponse,
WorldUploadResponse,
} from '@hexahost/contracts';
import type { GameServer, WorldUpload } from '@hexahost/database';
import {
createPresignedPutUrl,
createS3Client,
headStoredObject,
loadStorageConfig,
worldUploadObjectKey,
} from '@hexahost/storage';
import { AuditService } from '../../auth/audit.service';
import { NodeBridgeService } from '../../nodes/node-bridge.service';
import { PrismaService } from '../../prisma/prisma.service';
import { findOwnedServer } from '../shared/server-ownership.util';
import { ServerStateService } from '../server-state.service';
import { SERVER_BACKUPS_QUEUE } from '../backups/backups.service';
const WORLD_MANAGE_STATUSES = new Set<GameServer['status']>(['STOPPED']);
@Injectable()
export class WorldsService {
constructor(
private readonly prisma: PrismaService,
private readonly nodeBridge: NodeBridgeService,
private readonly serverState: ServerStateService,
private readonly auditService: AuditService,
@Inject(SERVER_BACKUPS_QUEUE) private readonly backupsQueue: Queue,
) {}
async getWorldInfo(userId: string, serverId: string): Promise<WorldInfoResponse> {
const server = await this.requireNode(serverId, userId);
const response = await this.nodeBridge.sendAgentRequest(
server.nodeId!,
'server.world.validate',
{
serverId,
generation: server.version,
worldName: server.activeWorldName,
},
);
if (!response.success) {
throw new ServiceUnavailableException(
response.errorMessage ?? 'Failed to load world info',
);
}
const result = response.result as {
worlds?: Array<{
name: string;
path: string;
sizeBytes: number;
hasLevelDat: boolean;
}>;
};
return {
activeWorldName: server.activeWorldName,
worlds: (result.worlds ?? []).map((world) => ({
name: world.name,
path: world.path,
sizeBytes: String(world.sizeBytes),
hasLevelDat: world.hasLevelDat,
})),
};
}
async initiateUpload(
userId: string,
serverId: string,
input: InitiateWorldUpload,
): Promise<WorldUploadInitResponse> {
const server = await this.requireStoppedServer(userId, serverId);
const uploadId = randomUUID();
const config = loadStorageConfig();
const client = createS3Client(config);
const s3Key = worldUploadObjectKey(serverId, uploadId);
const expiresAt = new Date(Date.now() + 60 * 60 * 1000);
const uploadUrl = await createPresignedPutUrl(
client,
config.S3_BUCKET,
s3Key,
60 * 60,
'application/gzip',
);
await this.prisma.worldUpload.create({
data: {
id: uploadId,
serverId,
status: 'UPLOADING',
s3Key,
worldName: input.worldName,
},
});
return {
uploadId,
uploadUrl,
expiresAt: expiresAt.toISOString(),
s3Key,
};
}
async completeUpload(
userId: string,
serverId: string,
uploadId: string,
): Promise<WorldUploadResponse> {
const server = await this.requireStoppedServer(userId, serverId);
const upload = await this.prisma.worldUpload.findFirstOrThrow({
where: { id: uploadId, serverId },
});
const config = loadStorageConfig();
const client = createS3Client(config);
const head = await headStoredObject(client, config.S3_BUCKET, upload.s3Key);
if (head.sizeBytes <= 0) {
throw new BadRequestException('Uploaded world archive is empty');
}
const updated = await this.prisma.worldUpload.update({
where: { id: uploadId },
data: {
status: 'READY',
sizeBytes: BigInt(head.sizeBytes),
},
});
await this.backupsQueue.add('apply-world-upload', {
uploadId,
serverId,
});
await this.auditService.record({
action: 'server.world.upload',
userId,
entityType: 'game_server',
entityId: serverId,
metadata: { uploadId, worldName: upload.worldName },
});
return this.toUploadResponse(updated);
}
async requestWorldDownload(
userId: string,
serverId: string,
): Promise<WorldDownloadUrlResponse> {
const server = await this.requireNode(serverId, userId);
if (server.status !== 'RUNNING' && server.status !== 'STOPPED') {
throw new BadRequestException(
`Cannot download world while server is ${server.status}`,
);
}
const backup = await this.prisma.serverBackup.create({
data: {
serverId,
type: 'MANUAL',
status: 'PENDING',
label: 'World download',
},
});
const previousStatus = server.status as 'RUNNING' | 'STOPPED';
await this.prisma.gameServer.update({
where: { id: serverId },
data: { status: 'BACKING_UP' },
});
await this.prisma.gameServerStateTransition.create({
data: {
gameServerId: serverId,
fromStatus: server.status,
toStatus: 'BACKING_UP',
reason: 'World download backup started',
actorId: userId,
},
});
await this.backupsQueue.add('create-backup', {
backupId: backup.id,
serverId,
previousStatus,
});
return {
backupId: backup.id,
downloadUrl: '',
expiresAt: new Date(Date.now() + 15 * 60 * 1000).toISOString(),
};
}
private async requireNode(serverId: string, userId: string): Promise<GameServer> {
const server = await findOwnedServer(this.prisma, userId, serverId);
if (!server.nodeId) {
throw new BadRequestException('Server is not assigned to a node');
}
return server;
}
private async requireStoppedServer(
userId: string,
serverId: string,
): Promise<GameServer> {
const server = await this.requireNode(serverId, userId);
this.serverState.assertNotInProgress(server.status);
if (!WORLD_MANAGE_STATUSES.has(server.status)) {
throw new BadRequestException(
`World changes require server to be STOPPED (current: ${server.status})`,
);
}
return server;
}
private toUploadResponse(upload: WorldUpload): WorldUploadResponse {
return {
id: upload.id,
serverId: upload.serverId,
status: upload.status,
worldName: upload.worldName,
sizeBytes: upload.sizeBytes?.toString() ?? null,
failureReason: upload.failureReason,
createdAt: upload.createdAt.toISOString(),
completedAt: upload.completedAt?.toISOString() ?? null,
};
}
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,159 @@
package archive
import (
"archive/tar"
"compress/gzip"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"strings"
)
// CreateTarGz archives sourceDir into outputPath and returns checksum and size.
func CreateTarGz(sourceDir, outputPath string) (sha256sum string, size int64, err error) {
sourceDir = filepath.Clean(sourceDir)
info, err := os.Stat(sourceDir)
if err != nil {
return "", 0, err
}
if !info.IsDir() {
return "", 0, fmt.Errorf("source is not a directory")
}
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
return "", 0, err
}
file, err := os.Create(outputPath)
if err != nil {
return "", 0, err
}
defer file.Close()
hasher := sha256.New()
writer := io.MultiWriter(file, hasher)
gzipWriter := gzip.NewWriter(writer)
tarWriter := tar.NewWriter(gzipWriter)
err = filepath.Walk(sourceDir, func(path string, entry os.FileInfo, walkErr error) error {
if walkErr != nil {
return walkErr
}
rel, err := filepath.Rel(sourceDir, path)
if err != nil {
return err
}
if rel == "." {
return nil
}
rel = filepath.ToSlash(rel)
header, err := tar.FileInfoHeader(entry, "")
if err != nil {
return err
}
header.Name = rel
if err := tarWriter.WriteHeader(header); err != nil {
return err
}
if entry.IsDir() {
return nil
}
sourceFile, err := os.Open(path)
if err != nil {
return err
}
defer sourceFile.Close()
_, err = io.Copy(tarWriter, sourceFile)
return err
})
if err != nil {
return "", 0, err
}
if err := tarWriter.Close(); err != nil {
return "", 0, err
}
if err := gzipWriter.Close(); err != nil {
return "", 0, err
}
stat, err := file.Stat()
if err != nil {
return "", 0, err
}
return hex.EncodeToString(hasher.Sum(nil)), stat.Size(), nil
}
// ExtractTarGz extracts archivePath into destDir.
func ExtractTarGz(archivePath, destDir string) error {
destDir = filepath.Clean(destDir)
if err := os.MkdirAll(destDir, 0o755); err != nil {
return err
}
file, err := os.Open(archivePath)
if err != nil {
return err
}
defer file.Close()
gzipReader, err := gzip.NewReader(file)
if err != nil {
return err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
target := filepath.Join(destDir, filepath.FromSlash(header.Name))
if !strings.HasPrefix(filepath.Clean(target), destDir+string(os.PathSeparator)) && filepath.Clean(target) != destDir {
return fmt.Errorf("archive entry escapes destination: %s", header.Name)
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode))
if err != nil {
return err
}
if _, err := io.Copy(out, tarReader); err != nil {
out.Close()
return err
}
out.Close()
default:
continue
}
}
return nil
}

View File

@@ -40,6 +40,10 @@ const (
TypeServerFilesWrite MessageType = "server.files.write"
TypeServerFilesUploadDone MessageType = "server.files.upload.complete"
TypeServerWorldValidate MessageType = "server.world.validate"
TypeServerWorldArchive MessageType = "server.world.archive"
TypeServerWorldReplace MessageType = "server.world.replace"
TypeServerStorageUpload MessageType = "server.storage.upload"
TypeServerStorageDownload MessageType = "server.storage.download"
TypeServerMetricsSub MessageType = "server.metrics.subscribe"
TypeServerLogsSubscribe MessageType = "server.logs.subscribe"
)
@@ -72,6 +76,10 @@ var controlToAgentTypes = map[MessageType]struct{}{
TypeServerFilesWrite: {},
TypeServerFilesUploadDone: {},
TypeServerWorldValidate: {},
TypeServerWorldArchive: {},
TypeServerWorldReplace: {},
TypeServerStorageUpload: {},
TypeServerStorageDownload: {},
TypeServerMetricsSub: {},
TypeServerLogsSubscribe: {},
}
@@ -178,6 +186,40 @@ type ServerFilesWritePayload struct {
Create bool `json:"create"`
}
// ServerWorldValidatePayload validates a Minecraft world directory.
type ServerWorldValidatePayload struct {
OperationMeta
WorldName string `json:"worldName"`
}
// ServerWorldArchivePayload archives a world directory.
type ServerWorldArchivePayload struct {
OperationMeta
WorldName string `json:"worldName"`
ArchiveID string `json:"archiveId"`
}
// ServerStorageUploadPayload uploads a local file to a presigned URL.
type ServerStorageUploadPayload struct {
OperationMeta
LocalPath string `json:"localPath"`
UploadURL string `json:"uploadUrl"`
}
// ServerStorageDownloadPayload downloads a remote file to a local path.
type ServerStorageDownloadPayload struct {
OperationMeta
DownloadURL string `json:"downloadUrl"`
LocalPath string `json:"localPath"`
}
// ServerWorldReplacePayload replaces a world from a local archive.
type ServerWorldReplacePayload struct {
OperationMeta
WorldName string `json:"worldName"`
ArchivePath string `json:"archivePath"`
}
// FileEntry describes a file or directory in a list response.
type FileEntry struct {
Name string `json:"name"`

View File

@@ -5,11 +5,16 @@ import (
"encoding/json"
"errors"
"log/slog"
"os"
"path/filepath"
"sync"
"github.com/hexahost/gamecloud/node-agent/internal/archive"
"github.com/hexahost/gamecloud/node-agent/internal/docker"
"github.com/hexahost/gamecloud/node-agent/internal/files"
"github.com/hexahost/gamecloud/node-agent/internal/protocol"
"github.com/hexahost/gamecloud/node-agent/internal/transfer"
worldpkg "github.com/hexahost/gamecloud/node-agent/internal/world"
)
const (
@@ -352,6 +357,219 @@ func (m *Manager) HandleFilesWrite(ctx context.Context, correlationID string, pa
})
}
// HandleBackupPrepare runs save-off and save-all flush via RCON.
func (m *Manager) HandleBackupPrepare(ctx context.Context, correlationID string, meta protocol.OperationMeta) {
rec, ok := m.getServer(meta.ServerID)
if !ok {
m.failOperation(ctx, correlationID, meta, "backup.prepare", "SERVER_NOT_FOUND", "server is not provisioned on this node")
return
}
if rec.State != ServerStateRunning {
m.completeOperation(ctx, correlationID, meta, "backup.prepare", map[string]string{"skipped": "not_running"})
return
}
for _, command := range []string{"save-off", "save-all flush"} {
if _, _, err := m.docker.ExecRcon(ctx, rec.ContainerID, command); err != nil {
m.failOperation(ctx, correlationID, meta, "backup.prepare", "BACKUP_PREPARE_FAILED", err.Error())
return
}
}
m.completeOperation(ctx, correlationID, meta, "backup.prepare", map[string]string{"status": "ok"})
}
// HandleBackupRelease runs save-on via RCON.
func (m *Manager) HandleBackupRelease(ctx context.Context, correlationID string, meta protocol.OperationMeta) {
rec, ok := m.getServer(meta.ServerID)
if !ok {
m.failOperation(ctx, correlationID, meta, "backup.release", "SERVER_NOT_FOUND", "server is not provisioned on this node")
return
}
if rec.State != ServerStateRunning {
m.completeOperation(ctx, correlationID, meta, "backup.release", map[string]string{"skipped": "not_running"})
return
}
if _, _, err := m.docker.ExecRcon(ctx, rec.ContainerID, "save-on"); err != nil {
m.failOperation(ctx, correlationID, meta, "backup.release", "BACKUP_RELEASE_FAILED", err.Error())
return
}
m.completeOperation(ctx, correlationID, meta, "backup.release", map[string]string{"status": "ok"})
}
// HandleWorldValidate validates a world directory and lists worlds.
func (m *Manager) HandleWorldValidate(ctx context.Context, correlationID string, payload protocol.ServerWorldValidatePayload) {
meta := payload.OperationMeta
rec, ok := m.getServer(meta.ServerID)
if !ok {
m.failOperation(ctx, correlationID, meta, "world.validate", "SERVER_NOT_FOUND", "server is not provisioned on this node")
return
}
worldName := payload.WorldName
if worldName == "" {
worldName = "world"
}
if err := worldpkg.ValidateWorld(rec.DataPath, worldName); err != nil {
m.failOperation(ctx, correlationID, meta, "world.validate", "WORLD_INVALID", err.Error())
return
}
worlds, err := worldpkg.ListWorlds(rec.DataPath, worldName)
if err != nil {
m.failOperation(ctx, correlationID, meta, "world.validate", "WORLD_LIST_FAILED", err.Error())
return
}
m.completeOperation(ctx, correlationID, meta, "world.validate", map[string]any{
"valid": true,
"worldName": worldName,
"worlds": worlds,
})
}
// HandleWorldArchive creates a tar.gz archive of a world directory.
func (m *Manager) HandleWorldArchive(ctx context.Context, correlationID string, payload protocol.ServerWorldArchivePayload) {
meta := payload.OperationMeta
rec, ok := m.getServer(meta.ServerID)
if !ok {
m.failOperation(ctx, correlationID, meta, "world.archive", "SERVER_NOT_FOUND", "server is not provisioned on this node")
return
}
worldName := payload.WorldName
if worldName == "" {
worldName = "world"
}
if err := worldpkg.ValidateWorld(rec.DataPath, worldName); err != nil {
m.failOperation(ctx, correlationID, meta, "world.archive", "WORLD_INVALID", err.Error())
return
}
archiveDir := filepath.Join(rec.DataPath, ".hgc", "archives")
archiveID := payload.ArchiveID
if archiveID == "" {
archiveID = meta.ServerID
}
outputPath := filepath.Join(archiveDir, archiveID+".tar.gz")
sourceDir := filepath.Join(rec.DataPath, worldName)
checksum, size, err := archive.CreateTarGz(sourceDir, outputPath)
if err != nil {
m.failOperation(ctx, correlationID, meta, "world.archive", "ARCHIVE_FAILED", err.Error())
return
}
m.completeOperation(ctx, correlationID, meta, "world.archive", map[string]any{
"localPath": outputPath,
"sha256": checksum,
"sizeBytes": size,
"worldName": worldName,
})
}
// HandleStorageUpload uploads a local archive to object storage.
func (m *Manager) HandleStorageUpload(ctx context.Context, correlationID string, payload protocol.ServerStorageUploadPayload) {
meta := payload.OperationMeta
if err := transfer.UploadFile(payload.LocalPath, payload.UploadURL); err != nil {
m.failOperation(ctx, correlationID, meta, "storage.upload", "UPLOAD_FAILED", err.Error())
return
}
m.completeOperation(ctx, correlationID, meta, "storage.upload", map[string]string{
"localPath": payload.LocalPath,
})
}
// HandleStorageDownload downloads an archive from object storage.
func (m *Manager) HandleStorageDownload(ctx context.Context, correlationID string, payload protocol.ServerStorageDownloadPayload) {
meta := payload.OperationMeta
if err := transfer.DownloadFile(payload.DownloadURL, payload.LocalPath); err != nil {
m.failOperation(ctx, correlationID, meta, "storage.download", "DOWNLOAD_FAILED", err.Error())
return
}
m.completeOperation(ctx, correlationID, meta, "storage.download", map[string]string{
"localPath": payload.LocalPath,
})
}
// HandleWorldReplace replaces a world directory from a local archive.
func (m *Manager) HandleWorldReplace(ctx context.Context, correlationID string, payload protocol.ServerWorldReplacePayload) {
meta := payload.OperationMeta
rec, ok := m.getServer(meta.ServerID)
if !ok {
m.failOperation(ctx, correlationID, meta, "world.replace", "SERVER_NOT_FOUND", "server is not provisioned on this node")
return
}
if rec.State == ServerStateRunning {
m.failOperation(ctx, correlationID, meta, "world.replace", "SERVER_RUNNING", "server must be stopped before world replace")
return
}
worldName := payload.WorldName
if worldName == "" {
worldName = "world"
}
worldDir := filepath.Join(rec.DataPath, worldName)
tempDir := filepath.Join(rec.DataPath, ".hgc", "restore", meta.ServerID)
if err := os.RemoveAll(tempDir); err != nil {
m.failOperation(ctx, correlationID, meta, "world.replace", "RESTORE_PREP_FAILED", err.Error())
return
}
if err := os.MkdirAll(tempDir, 0o755); err != nil {
m.failOperation(ctx, correlationID, meta, "world.replace", "RESTORE_PREP_FAILED", err.Error())
return
}
if err := archive.ExtractTarGz(payload.ArchivePath, tempDir); err != nil {
m.failOperation(ctx, correlationID, meta, "world.replace", "EXTRACT_FAILED", err.Error())
return
}
extractedWorld := filepath.Join(tempDir, worldName)
if info, err := os.Stat(extractedWorld); err != nil || !info.IsDir() {
extractedWorld = tempDir
}
if err := worldpkg.ValidateWorld(filepath.Dir(extractedWorld), filepath.Base(extractedWorld)); err != nil {
m.failOperation(ctx, correlationID, meta, "world.replace", "WORLD_INVALID", err.Error())
return
}
backupDir := worldDir + ".bak"
_ = os.RemoveAll(backupDir)
if _, err := os.Stat(worldDir); err == nil {
if err := os.Rename(worldDir, backupDir); err != nil {
m.failOperation(ctx, correlationID, meta, "world.replace", "REPLACE_FAILED", err.Error())
return
}
}
if err := os.Rename(extractedWorld, worldDir); err != nil {
_ = os.Rename(backupDir, worldDir)
m.failOperation(ctx, correlationID, meta, "world.replace", "REPLACE_FAILED", err.Error())
return
}
_ = os.RemoveAll(tempDir)
_ = os.RemoveAll(backupDir)
m.completeOperation(ctx, correlationID, meta, "world.replace", map[string]string{
"worldName": worldName,
})
}
func (m *Manager) getServer(serverID string) (*serverRecord, bool) {
m.mu.RLock()
defer m.mu.RUnlock()

View File

@@ -0,0 +1,73 @@
package transfer
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"time"
)
// UploadFile PUTs a local file to a presigned URL.
func UploadFile(localPath, uploadURL string) error {
file, err := os.Open(localPath)
if err != nil {
return err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return err
}
request, err := http.NewRequest(http.MethodPut, uploadURL, file)
if err != nil {
return err
}
request.ContentLength = stat.Size()
request.Header.Set("Content-Type", "application/gzip")
client := &http.Client{Timeout: 30 * time.Minute}
response, err := client.Do(request)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return fmt.Errorf("upload failed with status %d: %s", response.StatusCode, string(body))
}
return nil
}
// DownloadFile GETs a remote file to a local path.
func DownloadFile(downloadURL, localPath string) error {
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
return err
}
client := &http.Client{Timeout: 30 * time.Minute}
response, err := client.Get(downloadURL)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return fmt.Errorf("download failed with status %d: %s", response.StatusCode, string(body))
}
file, err := os.Create(localPath)
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, response.Body)
return err
}

View File

@@ -0,0 +1,114 @@
package world
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// Summary describes a discovered Minecraft world directory.
type Summary struct {
Name string `json:"name"`
Path string `json:"path"`
SizeBytes int64 `json:"sizeBytes"`
HasLevelDat bool `json:"hasLevelDat"`
}
// ValidateWorld checks that a world directory contains level.dat.
func ValidateWorld(dataPath, worldName string) error {
worldName = strings.TrimSpace(worldName)
if worldName == "" || strings.Contains(worldName, "..") || strings.ContainsAny(worldName, `/\`) {
return fmt.Errorf("invalid world name")
}
worldDir := filepath.Join(dataPath, worldName)
levelDat := filepath.Join(worldDir, "level.dat")
info, err := os.Stat(levelDat)
if err != nil {
return fmt.Errorf("level.dat not found in world directory")
}
if info.IsDir() {
return fmt.Errorf("level.dat is not a file")
}
return nil
}
// ListWorlds scans the server data directory for world folders.
func ListWorlds(dataPath, activeWorldName string) ([]Summary, error) {
entries, err := os.ReadDir(dataPath)
if err != nil {
return nil, err
}
worlds := make([]Summary, 0)
for _, entry := range entries {
if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") {
continue
}
worldPath := filepath.Join(dataPath, entry.Name())
levelDat := filepath.Join(worldPath, "level.dat")
hasLevelDat := false
if info, err := os.Stat(levelDat); err == nil && !info.IsDir() {
hasLevelDat = true
}
size, err := dirSize(worldPath)
if err != nil {
return nil, err
}
worlds = append(worlds, Summary{
Name: entry.Name(),
Path: entry.Name(),
SizeBytes: size,
HasLevelDat: hasLevelDat,
})
}
if activeWorldName != "" {
found := false
for _, world := range worlds {
if world.Name == activeWorldName {
found = true
break
}
}
if !found {
if err := ValidateWorld(dataPath, activeWorldName); err == nil {
size, sizeErr := dirSize(filepath.Join(dataPath, activeWorldName))
if sizeErr != nil {
return nil, sizeErr
}
worlds = append(worlds, Summary{
Name: activeWorldName,
Path: activeWorldName,
SizeBytes: size,
HasLevelDat: true,
})
}
}
}
return worlds, nil
}
func dirSize(root string) (int64, error) {
var total int64
err := filepath.Walk(root, func(_ string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
total += info.Size()
}
return nil
})
return total, err
}

View File

@@ -314,6 +314,62 @@ func (c *Client) dispatch(ctx context.Context, env *protocol.Envelope) {
}
c.runtime.HandleFilesWrite(ctx, env.MessageID, payload)
case protocol.TypeServerBackupPrepare:
var meta protocol.OperationMeta
if err := json.Unmarshal(env.Payload, &meta); err != nil {
c.log.Warn("decode server.backup.prepare", "error", err)
return
}
c.runtime.HandleBackupPrepare(ctx, env.MessageID, meta)
case protocol.TypeServerBackupRelease:
var meta protocol.OperationMeta
if err := json.Unmarshal(env.Payload, &meta); err != nil {
c.log.Warn("decode server.backup.release", "error", err)
return
}
c.runtime.HandleBackupRelease(ctx, env.MessageID, meta)
case protocol.TypeServerWorldValidate:
var payload protocol.ServerWorldValidatePayload
if err := json.Unmarshal(env.Payload, &payload); err != nil {
c.log.Warn("decode server.world.validate", "error", err)
return
}
c.runtime.HandleWorldValidate(ctx, env.MessageID, payload)
case protocol.TypeServerWorldArchive:
var payload protocol.ServerWorldArchivePayload
if err := json.Unmarshal(env.Payload, &payload); err != nil {
c.log.Warn("decode server.world.archive", "error", err)
return
}
c.runtime.HandleWorldArchive(ctx, env.MessageID, payload)
case protocol.TypeServerStorageUpload:
var payload protocol.ServerStorageUploadPayload
if err := json.Unmarshal(env.Payload, &payload); err != nil {
c.log.Warn("decode server.storage.upload", "error", err)
return
}
c.runtime.HandleStorageUpload(ctx, env.MessageID, payload)
case protocol.TypeServerStorageDownload:
var payload protocol.ServerStorageDownloadPayload
if err := json.Unmarshal(env.Payload, &payload); err != nil {
c.log.Warn("decode server.storage.download", "error", err)
return
}
c.runtime.HandleStorageDownload(ctx, env.MessageID, payload)
case protocol.TypeServerWorldReplace:
var payload protocol.ServerWorldReplacePayload
if err := json.Unmarshal(env.Payload, &payload); err != nil {
c.log.Warn("decode server.world.replace", "error", err)
return
}
c.runtime.HandleWorldReplace(ctx, env.MessageID, payload)
default:
c.log.Debug("ignored control message", "type", env.Type)
}

View File

@@ -190,7 +190,9 @@
"console": "Konsole",
"files": "Dateien",
"properties": "Eigenschaften",
"players": "Spieler"
"players": "Spieler",
"worlds": "Welten",
"backups": "Backups"
},
"console": {
"loading": "Wird geladen…",
@@ -274,6 +276,47 @@
"errors": {
"loadFailed": "Spielerliste konnte nicht geladen werden."
}
},
"worlds": {
"loading": "Wird geladen…",
"loadError": "Weltinformationen konnten nicht geladen werden.",
"activeWorld": "Aktive Welt: {name}",
"stoppedHint": "Upload und Ersetzen erfordern einen gestoppten Server.",
"worldName": "Weltname",
"uploadArchive": "Welt-Archiv (.tar.gz)",
"uploadError": "Welt-Upload fehlgeschlagen.",
"download": "Welt herunterladen",
"downloadError": "Download konnte nicht gestartet werden.",
"downloadPending": "Backup wird erstellt. Versuchen Sie den Download in Kürze erneut.",
"empty": "Keine Welten gefunden.",
"validWorld": "Gültige Welt",
"invalidWorld": "Ungültig",
"active": "Aktiv"
},
"backups": {
"loading": "Wird geladen…",
"loadError": "Backups konnten nicht geladen werden.",
"create": "Backup erstellen",
"manualLabel": "Manuelles Backup",
"createError": "Backup konnte nicht erstellt werden.",
"download": "Download",
"downloadError": "Download-URL konnte nicht erstellt werden.",
"restore": "Wiederherstellen",
"restoreConfirm": "Welt wirklich aus diesem Backup wiederherstellen? Der Server muss gestoppt sein.",
"restoreError": "Wiederherstellung fehlgeschlagen.",
"delete": "Löschen",
"deleteConfirm": "Backup wirklich löschen?",
"deleteError": "Backup konnte nicht gelöscht werden.",
"enableSchedule": "Geplante Backups aktivieren",
"disableSchedule": "Geplante Backups deaktivieren",
"scheduleError": "Zeitplan konnte nicht aktualisiert werden.",
"scheduleInfo": "Intervall: alle {hours}h · Aufbewahrung: {retention} Backups",
"label": "Bezeichnung",
"status": "Status",
"size": "Größe",
"created": "Erstellt",
"actions": "Aktionen",
"empty": "Noch keine Backups vorhanden."
}
}
}

View File

@@ -190,7 +190,9 @@
"console": "Console",
"files": "Files",
"properties": "Properties",
"players": "Players"
"players": "Players",
"worlds": "Worlds",
"backups": "Backups"
},
"console": {
"loading": "Loading…",
@@ -274,6 +276,47 @@
"errors": {
"loadFailed": "Could not load player list."
}
},
"worlds": {
"loading": "Loading…",
"loadError": "Could not load world information.",
"activeWorld": "Active world: {name}",
"stoppedHint": "Upload and replace require a stopped server.",
"worldName": "World name",
"uploadArchive": "World archive (.tar.gz)",
"uploadError": "World upload failed.",
"download": "Download world",
"downloadError": "Could not start download.",
"downloadPending": "Backup is being created. Try the download again shortly.",
"empty": "No worlds found.",
"validWorld": "Valid world",
"invalidWorld": "Invalid",
"active": "Active"
},
"backups": {
"loading": "Loading…",
"loadError": "Could not load backups.",
"create": "Create backup",
"manualLabel": "Manual backup",
"createError": "Could not create backup.",
"download": "Download",
"downloadError": "Could not create download URL.",
"restore": "Restore",
"restoreConfirm": "Restore the world from this backup? The server must be stopped.",
"restoreError": "Restore failed.",
"delete": "Delete",
"deleteConfirm": "Delete this backup?",
"deleteError": "Could not delete backup.",
"enableSchedule": "Enable scheduled backups",
"disableSchedule": "Disable scheduled backups",
"scheduleError": "Could not update schedule.",
"scheduleInfo": "Interval: every {hours}h · Retention: {retention} backups",
"label": "Label",
"status": "Status",
"size": "Size",
"created": "Created",
"actions": "Actions",
"empty": "No backups yet."
}
}
}

View File

@@ -0,0 +1,13 @@
import { setRequestLocale } from "next-intl/server";
import { ServerBackups } from "@/components/servers/server-backups";
interface ServerBackupsPageProps {
params: Promise<{ locale: string; id: string }>;
}
export default async function ServerBackupsPage({ params }: ServerBackupsPageProps) {
const { locale, id } = await params;
setRequestLocale(locale);
return <ServerBackups serverId={id} />;
}

View File

@@ -0,0 +1,13 @@
import { setRequestLocale } from "next-intl/server";
import { ServerWorlds } from "@/components/servers/server-worlds";
interface ServerWorldsPageProps {
params: Promise<{ locale: string; id: string }>;
}
export default async function ServerWorldsPage({ params }: ServerWorldsPageProps) {
const { locale, id } = await params;
setRequestLocale(locale);
return <ServerWorlds serverId={id} />;
}

View File

@@ -0,0 +1,237 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { useTranslations } from "next-intl";
import { Button } from "@hexahost/ui";
import {
createBackup,
deleteBackup,
getBackupDownloadUrl,
getBackupSchedule,
listBackups,
restoreBackup,
updateBackupSchedule,
type BackupItem,
type BackupSchedule,
} from "@/lib/api/backups";
interface ServerBackupsProps {
serverId: string;
}
function formatBytes(value: string | null): string {
if (!value) {
return "—";
}
const bytes = Number(value);
if (Number.isNaN(bytes)) {
return value;
}
if (bytes < 1024) {
return `${bytes} B`;
}
if (bytes < 1024 * 1024) {
return `${(bytes / 1024).toFixed(1)} KB`;
}
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
export function ServerBackups({ serverId }: ServerBackupsProps) {
const t = useTranslations("servers.backups");
const [backups, setBackups] = useState<BackupItem[]>([]);
const [schedule, setSchedule] = useState<BackupSchedule | null>(null);
const [loading, setLoading] = useState(true);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const refresh = useCallback(async () => {
setLoading(true);
setError(null);
try {
const [backupList, scheduleData] = await Promise.all([
listBackups(serverId),
getBackupSchedule(serverId),
]);
setBackups(backupList.backups);
setSchedule(scheduleData);
} catch (err) {
setError(err instanceof Error ? err.message : t("loadError"));
} finally {
setLoading(false);
}
}, [serverId, t]);
useEffect(() => {
void refresh();
}, [refresh]);
async function handleCreateBackup() {
setBusy(true);
setError(null);
try {
await createBackup(serverId, t("manualLabel"));
await refresh();
} catch (err) {
setError(err instanceof Error ? err.message : t("createError"));
} finally {
setBusy(false);
}
}
async function handleDownload(backupId: string) {
try {
const { downloadUrl } = await getBackupDownloadUrl(serverId, backupId);
window.open(downloadUrl, "_blank", "noopener,noreferrer");
} catch (err) {
setError(err instanceof Error ? err.message : t("downloadError"));
}
}
async function handleRestore(backupId: string) {
if (!window.confirm(t("restoreConfirm"))) {
return;
}
setBusy(true);
try {
await restoreBackup(serverId, backupId);
await refresh();
} catch (err) {
setError(err instanceof Error ? err.message : t("restoreError"));
} finally {
setBusy(false);
}
}
async function handleDelete(backupId: string) {
if (!window.confirm(t("deleteConfirm"))) {
return;
}
setBusy(true);
try {
await deleteBackup(serverId, backupId);
await refresh();
} catch (err) {
setError(err instanceof Error ? err.message : t("deleteError"));
} finally {
setBusy(false);
}
}
async function toggleSchedule(enabled: boolean) {
if (!schedule) {
return;
}
setBusy(true);
try {
const updated = await updateBackupSchedule(serverId, { enabled });
setSchedule(updated);
} catch (err) {
setError(err instanceof Error ? err.message : t("scheduleError"));
} finally {
setBusy(false);
}
}
if (loading) {
return <p className="text-sm text-zinc-500">{t("loading")}</p>;
}
return (
<div className="space-y-6">
{error ? (
<p className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-sm text-red-700 dark:border-red-900 dark:bg-red-950 dark:text-red-300">
{error}
</p>
) : null}
<div className="flex flex-wrap items-center gap-3">
<Button onClick={() => void handleCreateBackup()} disabled={busy}>
{t("create")}
</Button>
{schedule ? (
<Button
variant="secondary"
onClick={() => void toggleSchedule(!schedule.enabled)}
disabled={busy}
>
{schedule.enabled ? t("disableSchedule") : t("enableSchedule")}
</Button>
) : null}
</div>
{schedule ? (
<p className="text-sm text-zinc-600 dark:text-zinc-400">
{t("scheduleInfo", {
hours: schedule.intervalHours,
retention: schedule.retentionCount,
})}
</p>
) : null}
<div className="overflow-x-auto rounded-lg border border-zinc-200 dark:border-zinc-800">
<table className="min-w-full divide-y divide-zinc-200 text-sm dark:divide-zinc-800">
<thead className="bg-zinc-50 dark:bg-zinc-900">
<tr>
<th className="px-4 py-2 text-left font-medium">{t("label")}</th>
<th className="px-4 py-2 text-left font-medium">{t("status")}</th>
<th className="px-4 py-2 text-left font-medium">{t("size")}</th>
<th className="px-4 py-2 text-left font-medium">{t("created")}</th>
<th className="px-4 py-2 text-right font-medium">{t("actions")}</th>
</tr>
</thead>
<tbody className="divide-y divide-zinc-200 dark:divide-zinc-800">
{backups.length === 0 ? (
<tr>
<td colSpan={5} className="px-4 py-6 text-center text-zinc-500">
{t("empty")}
</td>
</tr>
) : (
backups.map((backup) => (
<tr key={backup.id}>
<td className="px-4 py-2">{backup.label ?? backup.type}</td>
<td className="px-4 py-2">{backup.status}</td>
<td className="px-4 py-2">{formatBytes(backup.sizeBytes)}</td>
<td className="px-4 py-2">
{new Date(backup.createdAt).toLocaleString()}
</td>
<td className="px-4 py-2 text-right space-x-2">
{backup.status === "AVAILABLE" ? (
<>
<Button
size="sm"
variant="secondary"
onClick={() => void handleDownload(backup.id)}
>
{t("download")}
</Button>
<Button
size="sm"
variant="secondary"
onClick={() => void handleRestore(backup.id)}
disabled={busy}
>
{t("restore")}
</Button>
</>
) : null}
<Button
size="sm"
variant="ghost"
onClick={() => void handleDelete(backup.id)}
disabled={busy}
>
{t("delete")}
</Button>
</td>
</tr>
))
)}
</tbody>
</table>
</div>
</div>
);
}

View File

@@ -13,6 +13,8 @@ const tabs = [
{ key: "files", suffix: "/files" },
{ key: "properties", suffix: "/properties" },
{ key: "players", suffix: "/players" },
{ key: "worlds", suffix: "/worlds" },
{ key: "backups", suffix: "/backups" },
] as const;
export function ServerTabs({ serverId }: ServerTabsProps) {

View File

@@ -0,0 +1,172 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { useTranslations } from "next-intl";
import { Button } from "@hexahost/ui";
import {
completeWorldUpload,
getWorldInfo,
initiateWorldUpload,
requestWorldDownload,
type WorldSummary,
} from "@/lib/api/worlds";
import { getBackupDownloadUrl } from "@/lib/api/backups";
interface ServerWorldsProps {
serverId: string;
}
function formatBytes(value: string): string {
const bytes = Number(value);
if (Number.isNaN(bytes)) {
return value;
}
if (bytes < 1024 * 1024) {
return `${(bytes / 1024).toFixed(1)} KB`;
}
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
export function ServerWorlds({ serverId }: ServerWorldsProps) {
const t = useTranslations("servers.worlds");
const [activeWorldName, setActiveWorldName] = useState("world");
const [worlds, setWorlds] = useState<WorldSummary[]>([]);
const [loading, setLoading] = useState(true);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const [worldName, setWorldName] = useState("world");
const refresh = useCallback(async () => {
setLoading(true);
setError(null);
try {
const info = await getWorldInfo(serverId);
setActiveWorldName(info.activeWorldName);
setWorlds(info.worlds);
} catch (err) {
setError(err instanceof Error ? err.message : t("loadError"));
} finally {
setLoading(false);
}
}, [serverId, t]);
useEffect(() => {
void refresh();
}, [refresh]);
async function handleUpload(file: File) {
setBusy(true);
setError(null);
try {
const init = await initiateWorldUpload(serverId, worldName);
const response = await fetch(init.uploadUrl, {
method: "PUT",
body: file,
headers: { "Content-Type": "application/gzip" },
});
if (!response.ok) {
throw new Error(t("uploadError"));
}
await completeWorldUpload(serverId, init.uploadId);
await refresh();
} catch (err) {
setError(err instanceof Error ? err.message : t("uploadError"));
} finally {
setBusy(false);
}
}
async function handleDownload() {
setBusy(true);
setError(null);
try {
const { backupId } = await requestWorldDownload(serverId);
const { downloadUrl } = await getBackupDownloadUrl(serverId, backupId);
if (downloadUrl) {
window.open(downloadUrl, "_blank", "noopener,noreferrer");
} else {
setError(t("downloadPending"));
}
} catch (err) {
setError(err instanceof Error ? err.message : t("downloadError"));
} finally {
setBusy(false);
}
}
if (loading) {
return <p className="text-sm text-zinc-500">{t("loading")}</p>;
}
return (
<div className="space-y-6">
{error ? (
<p className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-sm text-red-700 dark:border-red-900 dark:bg-red-950 dark:text-red-300">
{error}
</p>
) : null}
<div className="rounded-lg border border-zinc-200 p-4 dark:border-zinc-800">
<p className="text-sm text-zinc-600 dark:text-zinc-400">
{t("activeWorld", { name: activeWorldName })}
</p>
<p className="mt-1 text-xs text-zinc-500">{t("stoppedHint")}</p>
</div>
<div className="flex flex-wrap gap-3 items-end">
<label className="block text-sm">
<span className="mb-1 block text-zinc-600 dark:text-zinc-400">
{t("worldName")}
</span>
<input
className="rounded-md border border-zinc-300 bg-white px-3 py-2 text-sm dark:border-zinc-700 dark:bg-zinc-950"
value={worldName}
onChange={(event) => setWorldName(event.target.value)}
/>
</label>
<label className="block text-sm">
<span className="mb-1 block text-zinc-600 dark:text-zinc-400">
{t("uploadArchive")}
</span>
<input
type="file"
accept=".tar.gz,.tgz,application/gzip"
disabled={busy}
onChange={(event) => {
const file = event.target.files?.[0];
if (file) {
void handleUpload(file);
}
}}
/>
</label>
<Button variant="secondary" onClick={() => void handleDownload()} disabled={busy}>
{t("download")}
</Button>
</div>
<ul className="divide-y divide-zinc-200 rounded-lg border border-zinc-200 dark:divide-zinc-800 dark:border-zinc-800">
{worlds.length === 0 ? (
<li className="px-4 py-6 text-center text-sm text-zinc-500">{t("empty")}</li>
) : (
worlds.map((world) => (
<li key={world.name} className="flex items-center justify-between px-4 py-3 text-sm">
<div>
<p className="font-medium">{world.name}</p>
<p className="text-zinc-500">
{world.hasLevelDat ? t("validWorld") : t("invalidWorld")} ·{" "}
{formatBytes(world.sizeBytes)}
</p>
</div>
{world.name === activeWorldName ? (
<span className="text-xs text-sky-600 dark:text-sky-400">{t("active")}</span>
) : null}
</li>
))
)}
</ul>
</div>
);
}

View File

@@ -0,0 +1,80 @@
import { apiFetch } from "./auth";
export interface BackupItem {
id: string;
serverId: string;
type: string;
status: string;
sizeBytes: string | null;
sha256: string | null;
label: string | null;
failureReason: string | null;
createdAt: string;
completedAt: string | null;
}
export interface BackupListResponse {
backups: BackupItem[];
}
export interface BackupSchedule {
enabled: boolean;
intervalHours: number;
retentionCount: number;
nextRunAt: string | null;
lastRunAt: string | null;
}
export async function listBackups(serverId: string): Promise<BackupListResponse> {
return apiFetch<BackupListResponse>(`/servers/${serverId}/backups`);
}
export async function createBackup(
serverId: string,
label?: string,
): Promise<{ backup: BackupItem; jobId?: string }> {
return apiFetch(`/servers/${serverId}/backups`, {
method: "POST",
body: JSON.stringify({ label }),
});
}
export async function deleteBackup(
serverId: string,
backupId: string,
): Promise<BackupItem> {
return apiFetch(`/servers/${serverId}/backups/${backupId}`, {
method: "DELETE",
});
}
export async function getBackupDownloadUrl(
serverId: string,
backupId: string,
): Promise<{ downloadUrl: string; expiresAt: string }> {
return apiFetch(`/servers/${serverId}/backups/${backupId}/download-url`);
}
export async function restoreBackup(
serverId: string,
backupId: string,
): Promise<unknown> {
return apiFetch(`/servers/${serverId}/backups/${backupId}/restore`, {
method: "POST",
body: JSON.stringify({ confirm: true }),
});
}
export async function getBackupSchedule(serverId: string): Promise<BackupSchedule> {
return apiFetch(`/servers/${serverId}/backups/schedule`);
}
export async function updateBackupSchedule(
serverId: string,
input: Partial<BackupSchedule>,
): Promise<BackupSchedule> {
return apiFetch(`/servers/${serverId}/backups/schedule`, {
method: "PATCH",
body: JSON.stringify(input),
});
}

View File

@@ -0,0 +1,48 @@
import { apiFetch } from "./auth";
export interface WorldSummary {
name: string;
path: string;
sizeBytes: string;
hasLevelDat: boolean;
}
export interface WorldInfoResponse {
activeWorldName: string;
worlds: WorldSummary[];
}
export async function getWorldInfo(serverId: string): Promise<WorldInfoResponse> {
return apiFetch(`/servers/${serverId}/worlds`);
}
export async function initiateWorldUpload(
serverId: string,
worldName: string,
): Promise<{
uploadId: string;
uploadUrl: string;
expiresAt: string;
}> {
return apiFetch(`/servers/${serverId}/worlds/uploads`, {
method: "POST",
body: JSON.stringify({ worldName }),
});
}
export async function completeWorldUpload(
serverId: string,
uploadId: string,
): Promise<unknown> {
return apiFetch(`/servers/${serverId}/worlds/uploads/${uploadId}/complete`, {
method: "POST",
});
}
export async function requestWorldDownload(
serverId: string,
): Promise<{ backupId: string }> {
return apiFetch(`/servers/${serverId}/worlds/download`, {
method: "POST",
});
}

View File

@@ -11,9 +11,10 @@
"typecheck": "tsc --noEmit",
"test": "node --test test/**/*.test.js"
},
"dependencies": {
"dependencies": {
"@hexahost/config": "workspace:*",
"@hexahost/database": "workspace:*",
"@hexahost/storage": "workspace:*",
"bullmq": "^5.34.10",
"ioredis": "^5.4.2",
"nodemailer": "^6.10.0",

View File

@@ -0,0 +1,54 @@
import { randomUUID } from 'node:crypto';
import { validateConfig } from '@hexahost/config';
import { logger } from './logger';
import {
publishNodeCommand,
waitForNodeResponse,
type NodeResponseMessage,
} from './redis';
export interface AgentRequestResult extends NodeResponseMessage {
result?: unknown;
}
export async function sendAgentRequest(
nodeId: string,
type: string,
payload: Record<string, unknown>,
timeoutMs = 120_000,
): Promise<AgentRequestResult> {
const messageId = randomUUID();
await publishNodeCommand({
nodeId,
envelope: {
protocolVersion: 1,
messageId,
type,
timestamp: new Date().toISOString(),
payload: {
...payload,
generation: payload['generation'] ?? 0,
},
},
});
const response = await waitForNodeResponse(messageId, timeoutMs);
if (response === null) {
logger.error({ nodeId, type, messageId }, 'Agent request timed out');
return {
success: false,
errorCode: 'AGENT_TIMEOUT',
errorMessage: 'Agent request timed out',
};
}
return response as AgentRequestResult;
}
export function loadStorageEnv(): void {
validateConfig();
}

View File

@@ -0,0 +1,632 @@
import { Job } from 'bullmq';
import { z } from 'zod';
import { prisma } from '@hexahost/database';
import {
backupObjectKey,
createPresignedGetUrl,
createPresignedPutUrl,
createS3Client,
deleteStoredObject,
loadStorageConfig,
} from '@hexahost/storage';
import { sendAgentRequest } from '../agent-bridge';
import { logger } from '../logger';
import { transitionServerStatus } from '../server-state';
import { getBackupsQueue } from '../queues';
const BACKUP_TIMEOUT_MS = 20 * 60 * 1000;
const createBackupJobSchema = z.object({
backupId: z.string().uuid(),
serverId: z.string().uuid(),
previousStatus: z.enum(['RUNNING', 'STOPPED']),
});
const restoreBackupJobSchema = z.object({
restoreId: z.string().uuid(),
serverId: z.string().uuid(),
backupId: z.string().uuid(),
});
const applyWorldUploadJobSchema = z.object({
uploadId: z.string().uuid(),
serverId: z.string().uuid(),
});
const retentionJobSchema = z.object({
serverId: z.string().uuid(),
});
interface ArchiveResult {
localPath: string;
sha256: string;
sizeBytes: number;
worldName: string;
}
function getStorage() {
const config = loadStorageConfig();
return {
config,
client: createS3Client(config),
};
}
async function loadServer(serverId: string) {
const server = await prisma.gameServer.findUnique({
where: { id: serverId },
include: { backupSchedule: true },
});
if (!server?.nodeId) {
throw new Error(`Server ${serverId} is not assigned to a node`);
}
return server;
}
async function markBackupFailed(
backupId: string,
reason: string,
): Promise<void> {
await prisma.serverBackup.update({
where: { id: backupId },
data: {
status: 'FAILED',
failureReason: reason,
completedAt: new Date(),
},
});
}
async function runBackupArchive(
serverId: string,
nodeId: string,
backupId: string,
worldName: string,
generation: number,
): Promise<ArchiveResult> {
const response = await sendAgentRequest(
nodeId,
'server.world.archive',
{
serverId,
generation,
worldName,
archiveId: backupId,
},
BACKUP_TIMEOUT_MS,
);
if (!response.success) {
throw new Error(response.errorMessage ?? 'World archive failed');
}
return response.result as ArchiveResult;
}
async function uploadArchiveToS3(
serverId: string,
nodeId: string,
backupId: string,
archive: ArchiveResult,
generation: number,
): Promise<void> {
const { config, client } = getStorage();
const s3Key = backupObjectKey(serverId, backupId);
const uploadUrl = await createPresignedPutUrl(
client,
config.S3_BUCKET,
s3Key,
60 * 60,
);
const response = await sendAgentRequest(
nodeId,
'server.storage.upload',
{
serverId,
generation,
localPath: archive.localPath,
uploadUrl,
},
BACKUP_TIMEOUT_MS,
);
if (!response.success) {
throw new Error(response.errorMessage ?? 'Backup upload failed');
}
await prisma.serverBackup.update({
where: { id: backupId },
data: {
s3Key,
sha256: archive.sha256,
sizeBytes: BigInt(archive.sizeBytes),
},
});
}
async function enforceRetention(serverId: string, retentionCount: number): Promise<void> {
const backups = await prisma.serverBackup.findMany({
where: {
serverId,
status: 'AVAILABLE',
type: { in: ['MANUAL', 'SCHEDULED'] },
},
orderBy: { createdAt: 'desc' },
});
const extras = backups.slice(retentionCount);
if (extras.length === 0) {
return;
}
const { config, client } = getStorage();
for (const backup of extras) {
if (backup.s3Key) {
try {
await deleteStoredObject(client, config.S3_BUCKET, backup.s3Key);
} catch (error) {
logger.warn({ backupId: backup.id, error }, 'Failed to delete S3 backup object');
}
}
await prisma.serverBackup.update({
where: { id: backup.id },
data: { status: 'DELETED' },
});
}
}
export async function processCreateBackupJob(job: Job): Promise<{ status: string }> {
if (job.name !== 'create-backup') {
return { status: 'ignored' };
}
const { backupId, serverId, previousStatus } = createBackupJobSchema.parse(job.data);
const server = await loadServer(serverId);
await prisma.serverBackup.update({
where: { id: backupId },
data: { status: 'RUNNING' },
});
try {
if (previousStatus === 'RUNNING') {
const prepare = await sendAgentRequest(
server.nodeId!,
'server.backup.prepare',
{ serverId, generation: server.version },
BACKUP_TIMEOUT_MS,
);
if (!prepare.success) {
throw new Error(prepare.errorMessage ?? 'Backup prepare failed');
}
}
const archive = await runBackupArchive(
serverId,
server.nodeId!,
backupId,
server.activeWorldName,
server.version,
);
await uploadArchiveToS3(
serverId,
server.nodeId!,
backupId,
archive,
server.version,
);
if (previousStatus === 'RUNNING') {
const release = await sendAgentRequest(
server.nodeId!,
'server.backup.release',
{ serverId, generation: server.version },
BACKUP_TIMEOUT_MS,
);
if (!release.success) {
throw new Error(release.errorMessage ?? 'Backup release failed');
}
}
await prisma.serverBackup.update({
where: { id: backupId },
data: {
status: 'AVAILABLE',
completedAt: new Date(),
},
});
await transitionServerStatus(serverId, previousStatus, {
expectedFrom: 'BACKING_UP',
reason: 'Backup completed',
});
const retentionCount = server.backupSchedule?.retentionCount ?? 5;
await enforceRetention(serverId, retentionCount);
return { status: 'available' };
} catch (error) {
const message = error instanceof Error ? error.message : 'Backup failed';
await markBackupFailed(backupId, message);
try {
if (previousStatus === 'RUNNING') {
await sendAgentRequest(
server.nodeId!,
'server.backup.release',
{ serverId, generation: server.version },
60_000,
);
}
await transitionServerStatus(serverId, previousStatus, {
expectedFrom: 'BACKING_UP',
reason: 'Backup failed',
errorMessage: message,
});
} catch (restoreError) {
logger.error({ restoreError, serverId }, 'Failed to restore server status after backup error');
await transitionServerStatus(serverId, 'ERROR', {
expectedFrom: 'BACKING_UP',
reason: 'Backup failed and status recovery failed',
errorMessage: message,
});
}
throw error;
}
}
export async function processRestoreBackupJob(job: Job): Promise<{ status: string }> {
if (job.name !== 'restore-backup') {
return { status: 'ignored' };
}
const { restoreId, serverId, backupId } = restoreBackupJobSchema.parse(job.data);
const server = await loadServer(serverId);
const backup = await prisma.serverBackup.findUniqueOrThrow({
where: { id: backupId },
});
if (backup.status !== 'AVAILABLE' || !backup.s3Key) {
throw new Error('Backup is not available for restore');
}
await prisma.backupRestore.update({
where: { id: restoreId },
data: { status: 'RUNNING' },
});
const safetyBackup = await prisma.serverBackup.create({
data: {
serverId,
type: 'PRE_RESTORE',
status: 'RUNNING',
label: 'Pre-restore safety backup',
},
});
await prisma.backupRestore.update({
where: { id: restoreId },
data: { safetyBackupId: safetyBackup.id },
});
try {
const safetyArchive = await runBackupArchive(
serverId,
server.nodeId!,
safetyBackup.id,
server.activeWorldName,
server.version,
);
await uploadArchiveToS3(
serverId,
server.nodeId!,
safetyBackup.id,
safetyArchive,
server.version,
);
await prisma.serverBackup.update({
where: { id: safetyBackup.id },
data: {
status: 'AVAILABLE',
completedAt: new Date(),
},
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Safety backup failed';
await markBackupFailed(safetyBackup.id, message);
throw error;
}
const { config, client } = getStorage();
const downloadUrl = await createPresignedGetUrl(
client,
config.S3_BUCKET,
backup.s3Key,
60 * 60,
);
const localArchive = `/var/lib/hgc/servers/${serverId}/.hgc/restore/${restoreId}.tar.gz`;
try {
const download = await sendAgentRequest(
server.nodeId!,
'server.storage.download',
{
serverId,
generation: server.version,
downloadUrl,
localPath: localArchive,
},
BACKUP_TIMEOUT_MS,
);
if (!download.success) {
throw new Error(download.errorMessage ?? 'Backup download failed');
}
const replace = await sendAgentRequest(
server.nodeId!,
'server.world.replace',
{
serverId,
generation: server.version,
worldName: server.activeWorldName,
archivePath: localArchive,
},
BACKUP_TIMEOUT_MS,
);
if (!replace.success) {
throw new Error(replace.errorMessage ?? 'World replace failed');
}
await prisma.backupRestore.update({
where: { id: restoreId },
data: {
status: 'COMPLETED',
completedAt: new Date(),
},
});
await transitionServerStatus(serverId, 'STOPPED', {
expectedFrom: 'RESTORING',
reason: 'Restore completed',
});
return { status: 'completed' };
} catch (error) {
const message = error instanceof Error ? error.message : 'Restore failed';
await prisma.backupRestore.update({
where: { id: restoreId },
data: {
status: 'FAILED',
failureReason: message,
completedAt: new Date(),
},
});
await transitionServerStatus(serverId, 'STOPPED', {
expectedFrom: 'RESTORING',
reason: 'Restore failed',
errorMessage: message,
});
throw error;
}
}
export async function processApplyWorldUploadJob(job: Job): Promise<{ status: string }> {
if (job.name !== 'apply-world-upload') {
return { status: 'ignored' };
}
const { uploadId, serverId } = applyWorldUploadJobSchema.parse(job.data);
const server = await loadServer(serverId);
const upload = await prisma.worldUpload.findUniqueOrThrow({
where: { id: uploadId },
});
await prisma.worldUpload.update({
where: { id: uploadId },
data: { status: 'VALIDATING' },
});
const safetyBackup = await prisma.serverBackup.create({
data: {
serverId,
type: 'PRE_WORLD_REPLACE',
status: 'RUNNING',
label: `Pre-replace backup for upload ${uploadId}`,
},
});
try {
const safetyArchive = await runBackupArchive(
serverId,
server.nodeId!,
safetyBackup.id,
server.activeWorldName,
server.version,
);
await uploadArchiveToS3(
serverId,
server.nodeId!,
safetyBackup.id,
safetyArchive,
server.version,
);
await prisma.serverBackup.update({
where: { id: safetyBackup.id },
data: {
status: 'AVAILABLE',
completedAt: new Date(),
},
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Safety backup failed';
await markBackupFailed(safetyBackup.id, message);
throw error;
}
const { config, client } = getStorage();
const downloadUrl = await createPresignedGetUrl(
client,
config.S3_BUCKET,
upload.s3Key,
60 * 60,
);
const localArchive = `/var/lib/hgc/servers/${serverId}/.hgc/uploads/${uploadId}.tar.gz`;
try {
const download = await sendAgentRequest(
server.nodeId!,
'server.storage.download',
{
serverId,
generation: server.version,
downloadUrl,
localPath: localArchive,
},
BACKUP_TIMEOUT_MS,
);
if (!download.success) {
throw new Error(download.errorMessage ?? 'Upload download failed');
}
const replace = await sendAgentRequest(
server.nodeId!,
'server.world.replace',
{
serverId,
generation: server.version,
worldName: upload.worldName,
archivePath: localArchive,
},
BACKUP_TIMEOUT_MS,
);
if (!replace.success) {
throw new Error(replace.errorMessage ?? 'World replace failed');
}
await prisma.$transaction([
prisma.worldUpload.update({
where: { id: uploadId },
data: {
status: 'COMPLETED',
completedAt: new Date(),
},
}),
prisma.gameServer.update({
where: { id: serverId },
data: { activeWorldName: upload.worldName },
}),
]);
return { status: 'completed' };
} catch (error) {
const message = error instanceof Error ? error.message : 'World upload apply failed';
await prisma.worldUpload.update({
where: { id: uploadId },
data: {
status: 'FAILED',
failureReason: message,
completedAt: new Date(),
},
});
throw error;
}
}
export async function processRetentionJob(job: Job): Promise<{ status: string }> {
if (job.name !== 'enforce-retention') {
return { status: 'ignored' };
}
const { serverId } = retentionJobSchema.parse(job.data);
const server = await prisma.gameServer.findUnique({
where: { id: serverId },
include: { backupSchedule: true },
});
if (!server) {
return { status: 'ignored' };
}
await enforceRetention(serverId, server.backupSchedule?.retentionCount ?? 5);
return { status: 'ok' };
}
export async function processScheduledBackupsTick(): Promise<void> {
const now = new Date();
const schedules = await prisma.backupSchedule.findMany({
where: {
enabled: true,
OR: [{ nextRunAt: null }, { nextRunAt: { lte: now } }],
},
});
for (const schedule of schedules) {
const server = await prisma.gameServer.findUnique({
where: { id: schedule.serverId },
});
if (!server?.nodeId || !['RUNNING', 'STOPPED'].includes(server.status)) {
continue;
}
const backup = await prisma.serverBackup.create({
data: {
serverId: schedule.serverId,
type: 'SCHEDULED',
status: 'PENDING',
label: 'Scheduled backup',
},
});
await transitionServerStatus(schedule.serverId, 'BACKING_UP', {
expectedFrom: server.status as 'RUNNING' | 'STOPPED',
reason: 'Scheduled backup started',
});
await getBackupsQueue().add('create-backup', {
backupId: backup.id,
serverId: schedule.serverId,
previousStatus: server.status,
});
const nextRunAt = new Date(now.getTime() + schedule.intervalHours * 60 * 60 * 1000);
await prisma.backupSchedule.update({
where: { id: schedule.id },
data: {
lastRunAt: now,
nextRunAt,
},
});
}
}
export async function processBackupJob(job: Job): Promise<{ status: string }> {
switch (job.name) {
case 'create-backup':
return processCreateBackupJob(job);
case 'restore-backup':
return processRestoreBackupJob(job);
case 'apply-world-upload':
return processApplyWorldUploadJob(job);
case 'enforce-retention':
return processRetentionJob(job);
default:
logger.warn({ jobName: job.name }, 'Unknown backup job');
return { status: 'ignored' };
}
}

View File

@@ -4,6 +4,7 @@ import { validateConfig } from '@hexahost/config';
import { prisma } from '@hexahost/database';
import { processNotificationJob } from './handlers/notifications';
import { processBackupJob, processScheduledBackupsTick } from './handlers/server-backups';
import { processLifecycleJob } from './handlers/server-lifecycle';
import { processProvisionServerJob } from './handlers/server-provisioning';
import { startHealthServer } from './health';
@@ -11,6 +12,7 @@ import { logger } from './logger';
import { closeRedisConnections } from './redis';
import {
QUEUE_NOTIFICATIONS,
QUEUE_SERVER_BACKUPS,
QUEUE_SERVER_LIFECYCLE,
QUEUE_SERVER_PROVISIONING,
WORKER_QUEUES,
@@ -37,6 +39,10 @@ function createQueueWorker(
return processProvisionServerJob(job);
}
if (queueName === QUEUE_SERVER_BACKUPS) {
return processBackupJob(job);
}
if (queueName === QUEUE_SERVER_LIFECYCLE) {
return processLifecycleJob(job);
}
@@ -75,9 +81,17 @@ async function bootstrap(): Promise<void> {
logger.info({ queues: WORKER_QUEUES }, 'Worker started');
const scheduleTimer = setInterval(() => {
void processScheduledBackupsTick().catch((error: Error) => {
logger.error({ err: error }, 'Scheduled backup tick failed');
});
}, 60_000);
const shutdown = async (signal: string): Promise<void> => {
logger.info({ signal }, 'Shutting down worker');
clearInterval(scheduleTimer);
await Promise.all(workers.map((worker) => worker.close()));
await closeRedisConnections();
await prisma.$disconnect();

View File

@@ -1,11 +1,33 @@
export const QUEUE_SERVER_BACKUPS = 'server-backups' as const;
export const QUEUE_SERVER_LIFECYCLE = 'server-lifecycle' as const;
export const QUEUE_SERVER_PROVISIONING = 'server-provisioning' as const;
export const QUEUE_NOTIFICATIONS = 'notifications' as const;
import { Queue } from 'bullmq';
import { validateConfig } from '@hexahost/config';
export const WORKER_QUEUES = [
QUEUE_SERVER_BACKUPS,
QUEUE_SERVER_LIFECYCLE,
QUEUE_SERVER_PROVISIONING,
QUEUE_NOTIFICATIONS,
] as const;
export type WorkerQueueName = (typeof WORKER_QUEUES)[number];
let backupsQueue: Queue | null = null;
export function getBackupsQueue(): Queue {
if (!backupsQueue) {
const config = validateConfig();
backupsQueue = new Queue(QUEUE_SERVER_BACKUPS, {
connection: {
url: config.REDIS_URL,
maxRetriesPerRequest: null,
},
});
}
return backupsQueue;
}

View File

@@ -21,6 +21,7 @@ export interface NodeCommandMessage {
export interface NodeResponseMessage {
success: boolean;
result?: unknown;
resultCode?: string;
errorCode?: string;
errorMessage?: string;

File diff suppressed because one or more lines are too long