Phase4
This commit is contained in:
@@ -19,6 +19,7 @@
|
||||
"@hexahost/config": "workspace:*",
|
||||
"@hexahost/contracts": "workspace:*",
|
||||
"@hexahost/database": "workspace:*",
|
||||
"@hexahost/storage": "workspace:*",
|
||||
"@nestjs/common": "^11.0.7",
|
||||
"@nestjs/core": "^11.0.7",
|
||||
"@nestjs/platform-fastify": "^11.0.7",
|
||||
|
||||
@@ -320,7 +320,14 @@ export class NodesService {
|
||||
normalized === 'files.list' ||
|
||||
normalized === 'files.read' ||
|
||||
normalized === 'files.write' ||
|
||||
normalized === 'command'
|
||||
normalized === 'command' ||
|
||||
normalized === 'backup.prepare' ||
|
||||
normalized === 'backup.release' ||
|
||||
normalized === 'world.validate' ||
|
||||
normalized === 'world.archive' ||
|
||||
normalized === 'world.replace' ||
|
||||
normalized === 'storage.upload' ||
|
||||
normalized === 'storage.download'
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
102
apps/api/src/servers/backups/backups.controller.ts
Normal file
102
apps/api/src/servers/backups/backups.controller.ts
Normal file
@@ -0,0 +1,102 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
Param,
|
||||
ParseUUIDPipe,
|
||||
Patch,
|
||||
Post,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger';
|
||||
|
||||
import {
|
||||
createBackupRequestSchema,
|
||||
restoreBackupRequestSchema,
|
||||
updateBackupScheduleSchema,
|
||||
} from '@hexahost/contracts';
|
||||
|
||||
import { CurrentUser } from '../../auth/decorators/current-user.decorator';
|
||||
import { SessionGuard } from '../../auth/guards/session.guard';
|
||||
|
||||
import { BackupsService } from './backups.service';
|
||||
|
||||
@ApiTags('servers')
|
||||
@Controller('servers/:serverId/backups')
|
||||
@UseGuards(SessionGuard)
|
||||
export class BackupsController {
|
||||
constructor(private readonly backupsService: BackupsService) {}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({ summary: 'List server backups' })
|
||||
list(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
) {
|
||||
return this.backupsService.listBackups(user.id, serverId);
|
||||
}
|
||||
|
||||
@Post()
|
||||
@ApiOperation({ summary: 'Create a manual backup' })
|
||||
create(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Body() body: unknown,
|
||||
) {
|
||||
const input = createBackupRequestSchema.parse(body ?? {});
|
||||
return this.backupsService.createBackup(user.id, serverId, input);
|
||||
}
|
||||
|
||||
@Get('schedule')
|
||||
@ApiOperation({ summary: 'Get backup schedule' })
|
||||
getSchedule(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
) {
|
||||
return this.backupsService.getSchedule(user.id, serverId);
|
||||
}
|
||||
|
||||
@Patch('schedule')
|
||||
@ApiOperation({ summary: 'Update backup schedule' })
|
||||
updateSchedule(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Body() body: unknown,
|
||||
) {
|
||||
const input = updateBackupScheduleSchema.parse(body);
|
||||
return this.backupsService.updateSchedule(user.id, serverId, input);
|
||||
}
|
||||
|
||||
@Get(':backupId/download-url')
|
||||
@ApiOperation({ summary: 'Get a presigned download URL for a backup' })
|
||||
downloadUrl(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Param('backupId', ParseUUIDPipe) backupId: string,
|
||||
) {
|
||||
return this.backupsService.getDownloadUrl(user.id, serverId, backupId);
|
||||
}
|
||||
|
||||
@Post(':backupId/restore')
|
||||
@ApiOperation({ summary: 'Restore a backup' })
|
||||
restore(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Param('backupId', ParseUUIDPipe) backupId: string,
|
||||
@Body() body: unknown,
|
||||
) {
|
||||
restoreBackupRequestSchema.parse(body);
|
||||
return this.backupsService.restoreBackup(user.id, serverId, backupId);
|
||||
}
|
||||
|
||||
@Delete(':backupId')
|
||||
@ApiOperation({ summary: 'Delete a backup' })
|
||||
deleteBackup(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Param('backupId', ParseUUIDPipe) backupId: string,
|
||||
) {
|
||||
return this.backupsService.deleteBackup(user.id, serverId, backupId);
|
||||
}
|
||||
}
|
||||
34
apps/api/src/servers/backups/backups.module.ts
Normal file
34
apps/api/src/servers/backups/backups.module.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { getConfig } from '@hexahost/config';
|
||||
|
||||
import { AuthModule } from '../../auth/auth.module';
|
||||
import { PrismaModule } from '../../prisma/prisma.module';
|
||||
|
||||
import { BackupsController } from './backups.controller';
|
||||
import { BackupsService, SERVER_BACKUPS_QUEUE } from './backups.service';
|
||||
import { ServerStateService } from '../server-state.service';
|
||||
|
||||
@Module({
|
||||
imports: [AuthModule, PrismaModule],
|
||||
controllers: [BackupsController],
|
||||
providers: [
|
||||
BackupsService,
|
||||
ServerStateService,
|
||||
{
|
||||
provide: SERVER_BACKUPS_QUEUE,
|
||||
useFactory: () => {
|
||||
const config = getConfig();
|
||||
return new Queue('server-backups', {
|
||||
connection: {
|
||||
url: config.REDIS_URL,
|
||||
maxRetriesPerRequest: null,
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
],
|
||||
exports: [BackupsService, SERVER_BACKUPS_QUEUE],
|
||||
})
|
||||
export class BackupsModule {}
|
||||
341
apps/api/src/servers/backups/backups.service.ts
Normal file
341
apps/api/src/servers/backups/backups.service.ts
Normal file
@@ -0,0 +1,341 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Inject,
|
||||
Injectable,
|
||||
} from '@nestjs/common';
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import type { ServerBackup } from '@hexahost/database';
|
||||
import type {
|
||||
BackupActionResponse,
|
||||
BackupDownloadUrlResponse,
|
||||
BackupListResponse,
|
||||
BackupResponse,
|
||||
BackupRestoreResponse,
|
||||
BackupSchedule,
|
||||
CreateBackupRequest,
|
||||
UpdateBackupSchedule,
|
||||
} from '@hexahost/contracts';
|
||||
import {
|
||||
createPresignedGetUrl,
|
||||
createS3Client,
|
||||
deleteStoredObject,
|
||||
loadStorageConfig,
|
||||
} from '@hexahost/storage';
|
||||
|
||||
import { AuditService } from '../../auth/audit.service';
|
||||
import { PrismaService } from '../../prisma/prisma.service';
|
||||
import { findOwnedServer } from '../shared/server-ownership.util';
|
||||
import { ServerStateService } from '../server-state.service';
|
||||
|
||||
export const SERVER_BACKUPS_QUEUE = Symbol('SERVER_BACKUPS_QUEUE');
|
||||
|
||||
@Injectable()
|
||||
export class BackupsService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly serverState: ServerStateService,
|
||||
private readonly auditService: AuditService,
|
||||
@Inject(SERVER_BACKUPS_QUEUE) private readonly backupsQueue: Queue,
|
||||
) {}
|
||||
|
||||
async listBackups(userId: string, serverId: string): Promise<BackupListResponse> {
|
||||
await findOwnedServer(this.prisma, userId, serverId);
|
||||
|
||||
const backups = await this.prisma.serverBackup.findMany({
|
||||
where: {
|
||||
serverId,
|
||||
status: { not: 'DELETED' },
|
||||
},
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
|
||||
return { backups: backups.map((backup) => this.toResponse(backup)) };
|
||||
}
|
||||
|
||||
async createBackup(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
input: CreateBackupRequest,
|
||||
): Promise<BackupActionResponse> {
|
||||
const server = await findOwnedServer(this.prisma, userId, serverId);
|
||||
this.serverState.assertNotInProgress(server.status);
|
||||
|
||||
if (!server.nodeId) {
|
||||
throw new BadRequestException('Server is not assigned to a node');
|
||||
}
|
||||
|
||||
if (server.status !== 'RUNNING' && server.status !== 'STOPPED') {
|
||||
throw new BadRequestException(
|
||||
`Cannot create backup while server is ${server.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
const previousStatus = server.status;
|
||||
const backup = await this.prisma.serverBackup.create({
|
||||
data: {
|
||||
serverId,
|
||||
type: 'MANUAL',
|
||||
status: 'PENDING',
|
||||
label: input.label ?? 'Manual backup',
|
||||
},
|
||||
});
|
||||
|
||||
await this.prisma.gameServer.update({
|
||||
where: { id: serverId },
|
||||
data: { status: 'BACKING_UP' },
|
||||
});
|
||||
|
||||
await this.prisma.gameServerStateTransition.create({
|
||||
data: {
|
||||
gameServerId: serverId,
|
||||
fromStatus: previousStatus,
|
||||
toStatus: 'BACKING_UP',
|
||||
reason: 'Manual backup requested',
|
||||
actorId: userId,
|
||||
},
|
||||
});
|
||||
|
||||
const job = await this.backupsQueue.add('create-backup', {
|
||||
backupId: backup.id,
|
||||
serverId,
|
||||
previousStatus,
|
||||
});
|
||||
|
||||
await this.auditService.record({
|
||||
action: 'server.backup.create',
|
||||
userId,
|
||||
entityType: 'game_server',
|
||||
entityId: serverId,
|
||||
metadata: { backupId: backup.id },
|
||||
});
|
||||
|
||||
return {
|
||||
backup: this.toResponse(backup),
|
||||
jobId: job.id?.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
async getDownloadUrl(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
backupId: string,
|
||||
): Promise<BackupDownloadUrlResponse> {
|
||||
await findOwnedServer(this.prisma, userId, serverId);
|
||||
|
||||
const backup = await this.prisma.serverBackup.findFirstOrThrow({
|
||||
where: { id: backupId, serverId, status: 'AVAILABLE' },
|
||||
});
|
||||
|
||||
if (!backup.s3Key) {
|
||||
throw new BadRequestException('Backup is not stored yet');
|
||||
}
|
||||
|
||||
const config = loadStorageConfig();
|
||||
const client = createS3Client(config);
|
||||
const expiresAt = new Date(Date.now() + 15 * 60 * 1000);
|
||||
const downloadUrl = await createPresignedGetUrl(
|
||||
client,
|
||||
config.S3_BUCKET,
|
||||
backup.s3Key,
|
||||
15 * 60,
|
||||
);
|
||||
|
||||
return {
|
||||
downloadUrl,
|
||||
expiresAt: expiresAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
async deleteBackup(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
backupId: string,
|
||||
): Promise<BackupResponse> {
|
||||
await findOwnedServer(this.prisma, userId, serverId);
|
||||
|
||||
const backup = await this.prisma.serverBackup.findFirstOrThrow({
|
||||
where: { id: backupId, serverId },
|
||||
});
|
||||
|
||||
if (backup.s3Key) {
|
||||
const config = loadStorageConfig();
|
||||
const client = createS3Client(config);
|
||||
await deleteStoredObject(client, config.S3_BUCKET, backup.s3Key);
|
||||
}
|
||||
|
||||
const updated = await this.prisma.serverBackup.update({
|
||||
where: { id: backupId },
|
||||
data: { status: 'DELETED' },
|
||||
});
|
||||
|
||||
await this.auditService.record({
|
||||
action: 'server.backup.delete',
|
||||
userId,
|
||||
entityType: 'game_server',
|
||||
entityId: serverId,
|
||||
metadata: { backupId },
|
||||
});
|
||||
|
||||
return this.toResponse(updated);
|
||||
}
|
||||
|
||||
async restoreBackup(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
backupId: string,
|
||||
): Promise<BackupRestoreResponse> {
|
||||
const server = await findOwnedServer(this.prisma, userId, serverId);
|
||||
this.serverState.assertNotInProgress(server.status);
|
||||
|
||||
if (!server.nodeId) {
|
||||
throw new BadRequestException('Server is not assigned to a node');
|
||||
}
|
||||
|
||||
if (server.status === 'RUNNING') {
|
||||
throw new BadRequestException('Server must be stopped before restore');
|
||||
}
|
||||
|
||||
if (server.status !== 'STOPPED') {
|
||||
throw new BadRequestException(
|
||||
`Cannot restore while server is ${server.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
const backup = await this.prisma.serverBackup.findFirstOrThrow({
|
||||
where: { id: backupId, serverId, status: 'AVAILABLE' },
|
||||
});
|
||||
|
||||
const restore = await this.prisma.backupRestore.create({
|
||||
data: {
|
||||
serverId,
|
||||
backupId: backup.id,
|
||||
status: 'PENDING',
|
||||
},
|
||||
});
|
||||
|
||||
await this.prisma.gameServer.update({
|
||||
where: { id: serverId },
|
||||
data: { status: 'RESTORING' },
|
||||
});
|
||||
|
||||
await this.prisma.gameServerStateTransition.create({
|
||||
data: {
|
||||
gameServerId: serverId,
|
||||
fromStatus: 'STOPPED',
|
||||
toStatus: 'RESTORING',
|
||||
reason: 'Restore requested',
|
||||
actorId: userId,
|
||||
},
|
||||
});
|
||||
|
||||
const job = await this.backupsQueue.add('restore-backup', {
|
||||
restoreId: restore.id,
|
||||
serverId,
|
||||
backupId: backup.id,
|
||||
});
|
||||
|
||||
await this.auditService.record({
|
||||
action: 'server.backup.restore',
|
||||
userId,
|
||||
entityType: 'game_server',
|
||||
entityId: serverId,
|
||||
metadata: { backupId, restoreId: restore.id },
|
||||
});
|
||||
|
||||
return {
|
||||
restore: {
|
||||
id: restore.id,
|
||||
serverId,
|
||||
backupId: backup.id,
|
||||
safetyBackupId: restore.safetyBackupId,
|
||||
status: restore.status,
|
||||
failureReason: restore.failureReason,
|
||||
createdAt: restore.createdAt.toISOString(),
|
||||
completedAt: restore.completedAt?.toISOString() ?? null,
|
||||
},
|
||||
jobId: job.id?.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
async getSchedule(userId: string, serverId: string): Promise<BackupSchedule> {
|
||||
await findOwnedServer(this.prisma, userId, serverId);
|
||||
|
||||
const schedule = await this.prisma.backupSchedule.upsert({
|
||||
where: { serverId },
|
||||
create: {
|
||||
serverId,
|
||||
enabled: false,
|
||||
intervalHours: 24,
|
||||
retentionCount: 5,
|
||||
nextRunAt: new Date(Date.now() + 24 * 60 * 60 * 1000),
|
||||
},
|
||||
update: {},
|
||||
});
|
||||
|
||||
return this.toScheduleResponse(schedule);
|
||||
}
|
||||
|
||||
async updateSchedule(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
input: UpdateBackupSchedule,
|
||||
): Promise<BackupSchedule> {
|
||||
await findOwnedServer(this.prisma, userId, serverId);
|
||||
|
||||
const schedule = await this.prisma.backupSchedule.upsert({
|
||||
where: { serverId },
|
||||
create: {
|
||||
serverId,
|
||||
enabled: input.enabled ?? false,
|
||||
intervalHours: input.intervalHours ?? 24,
|
||||
retentionCount: input.retentionCount ?? 5,
|
||||
nextRunAt: new Date(Date.now() + (input.intervalHours ?? 24) * 60 * 60 * 1000),
|
||||
},
|
||||
update: {
|
||||
enabled: input.enabled,
|
||||
intervalHours: input.intervalHours,
|
||||
retentionCount: input.retentionCount,
|
||||
nextRunAt:
|
||||
input.enabled === true
|
||||
? new Date(Date.now() + (input.intervalHours ?? 24) * 60 * 60 * 1000)
|
||||
: undefined,
|
||||
},
|
||||
});
|
||||
|
||||
return this.toScheduleResponse(schedule);
|
||||
}
|
||||
|
||||
private toResponse(backup: ServerBackup): BackupResponse {
|
||||
return {
|
||||
id: backup.id,
|
||||
serverId: backup.serverId,
|
||||
type: backup.type,
|
||||
status: backup.status,
|
||||
s3Key: backup.s3Key,
|
||||
sizeBytes: backup.sizeBytes?.toString() ?? null,
|
||||
sha256: backup.sha256,
|
||||
label: backup.label,
|
||||
failureReason: backup.failureReason,
|
||||
createdAt: backup.createdAt.toISOString(),
|
||||
completedAt: backup.completedAt?.toISOString() ?? null,
|
||||
expiresAt: backup.expiresAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
private toScheduleResponse(schedule: {
|
||||
enabled: boolean;
|
||||
intervalHours: number;
|
||||
retentionCount: number;
|
||||
nextRunAt: Date | null;
|
||||
lastRunAt: Date | null;
|
||||
}): BackupSchedule {
|
||||
return {
|
||||
enabled: schedule.enabled,
|
||||
intervalHours: schedule.intervalHours,
|
||||
retentionCount: schedule.retentionCount,
|
||||
nextRunAt: schedule.nextRunAt?.toISOString() ?? null,
|
||||
lastRunAt: schedule.lastRunAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -7,10 +7,12 @@ const ALLOWED_TRANSITIONS: Record<GameServerStatus, GameServerStatus[]> = {
|
||||
DRAFT: ['PROVISIONING', 'DELETING'],
|
||||
PROVISIONING: ['INSTALLING', 'ERROR', 'DELETING'],
|
||||
INSTALLING: ['STOPPED', 'ERROR', 'DELETING'],
|
||||
STOPPED: ['STARTING', 'PROVISIONING', 'DELETING'],
|
||||
STOPPED: ['STARTING', 'PROVISIONING', 'BACKING_UP', 'RESTORING', 'DELETING'],
|
||||
STARTING: ['RUNNING', 'STOPPED', 'ERROR', 'DELETING'],
|
||||
RUNNING: ['STOPPING', 'ERROR', 'DELETING'],
|
||||
RUNNING: ['STOPPING', 'BACKING_UP', 'ERROR', 'DELETING'],
|
||||
STOPPING: ['STOPPED', 'ERROR', 'DELETING'],
|
||||
BACKING_UP: ['RUNNING', 'STOPPED', 'ERROR'],
|
||||
RESTORING: ['STOPPED', 'ERROR'],
|
||||
ERROR: ['STOPPED', 'STARTING', 'PROVISIONING', 'DELETING'],
|
||||
DELETING: ['DELETED'],
|
||||
DELETED: [],
|
||||
@@ -21,6 +23,8 @@ const IN_PROGRESS_STATUSES: GameServerStatus[] = [
|
||||
'INSTALLING',
|
||||
'STARTING',
|
||||
'STOPPING',
|
||||
'BACKING_UP',
|
||||
'RESTORING',
|
||||
'DELETING',
|
||||
];
|
||||
|
||||
|
||||
@@ -9,7 +9,9 @@ import { NodeBridgeModule } from '../nodes/node-bridge.module';
|
||||
import { AppConfigModule } from '../config/app-config.module';
|
||||
|
||||
import { ConsoleModule } from './console/console.module';
|
||||
import { BackupsModule } from './backups/backups.module';
|
||||
import { FilesModule } from './files/files.module';
|
||||
import { WorldsModule } from './worlds/worlds.module';
|
||||
import { PlayersModule } from './players/players.module';
|
||||
import { PropertiesModule } from './properties/properties.module';
|
||||
import { ServerStateService } from './server-state.service';
|
||||
@@ -28,6 +30,8 @@ import { PlansService } from './plans.service';
|
||||
AppConfigModule,
|
||||
AuthModule,
|
||||
forwardRef(() => ConsoleModule),
|
||||
forwardRef(() => BackupsModule),
|
||||
forwardRef(() => WorldsModule),
|
||||
forwardRef(() => FilesModule),
|
||||
forwardRef(() => PropertiesModule),
|
||||
forwardRef(() => PlayersModule),
|
||||
|
||||
@@ -314,6 +314,7 @@ export class ServersService {
|
||||
hostPort: server.hostPort,
|
||||
containerId: server.containerId,
|
||||
dataPath: server.dataPath,
|
||||
activeWorldName: server.activeWorldName,
|
||||
ramMb: server.ramMb,
|
||||
createdAt: server.createdAt.toISOString(),
|
||||
updatedAt: server.updatedAt.toISOString(),
|
||||
|
||||
63
apps/api/src/servers/worlds/worlds.controller.ts
Normal file
63
apps/api/src/servers/worlds/worlds.controller.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Get,
|
||||
Param,
|
||||
ParseUUIDPipe,
|
||||
Post,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
|
||||
import { initiateWorldUploadSchema } from '@hexahost/contracts';
|
||||
|
||||
import { CurrentUser } from '../../auth/decorators/current-user.decorator';
|
||||
import { SessionGuard } from '../../auth/guards/session.guard';
|
||||
|
||||
import { WorldsService } from './worlds.service';
|
||||
|
||||
@ApiTags('servers')
|
||||
@Controller('servers/:serverId/worlds')
|
||||
@UseGuards(SessionGuard)
|
||||
export class WorldsController {
|
||||
constructor(private readonly worldsService: WorldsService) {}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({ summary: 'Get world overview' })
|
||||
info(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
) {
|
||||
return this.worldsService.getWorldInfo(user.id, serverId);
|
||||
}
|
||||
|
||||
@Post('uploads')
|
||||
@ApiOperation({ summary: 'Initiate a world upload' })
|
||||
initiateUpload(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Body() body: unknown,
|
||||
) {
|
||||
const input = initiateWorldUploadSchema.parse(body);
|
||||
return this.worldsService.initiateUpload(user.id, serverId, input);
|
||||
}
|
||||
|
||||
@Post('uploads/:uploadId/complete')
|
||||
@ApiOperation({ summary: 'Complete a world upload and apply it' })
|
||||
completeUpload(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
@Param('uploadId', ParseUUIDPipe) uploadId: string,
|
||||
) {
|
||||
return this.worldsService.completeUpload(user.id, serverId, uploadId);
|
||||
}
|
||||
|
||||
@Post('download')
|
||||
@ApiOperation({ summary: 'Create a backup for world download' })
|
||||
download(
|
||||
@CurrentUser() user: { id: string },
|
||||
@Param('serverId', ParseUUIDPipe) serverId: string,
|
||||
) {
|
||||
return this.worldsService.requestWorldDownload(user.id, serverId);
|
||||
}
|
||||
}
|
||||
17
apps/api/src/servers/worlds/worlds.module.ts
Normal file
17
apps/api/src/servers/worlds/worlds.module.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { AuthModule } from '../../auth/auth.module';
|
||||
import { NodeBridgeModule } from '../../nodes/node-bridge.module';
|
||||
import { PrismaModule } from '../../prisma/prisma.module';
|
||||
import { BackupsModule } from '../backups/backups.module';
|
||||
import { ServerStateService } from '../server-state.service';
|
||||
|
||||
import { WorldsController } from './worlds.controller';
|
||||
import { WorldsService } from './worlds.service';
|
||||
|
||||
@Module({
|
||||
imports: [AuthModule, PrismaModule, NodeBridgeModule, BackupsModule],
|
||||
controllers: [WorldsController],
|
||||
providers: [WorldsService, ServerStateService],
|
||||
})
|
||||
export class WorldsModule {}
|
||||
250
apps/api/src/servers/worlds/worlds.service.ts
Normal file
250
apps/api/src/servers/worlds/worlds.service.ts
Normal file
@@ -0,0 +1,250 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Inject,
|
||||
Injectable,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { Queue } from 'bullmq';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import type {
|
||||
InitiateWorldUpload,
|
||||
WorldDownloadUrlResponse,
|
||||
WorldInfoResponse,
|
||||
WorldUploadInitResponse,
|
||||
WorldUploadResponse,
|
||||
} from '@hexahost/contracts';
|
||||
import type { GameServer, WorldUpload } from '@hexahost/database';
|
||||
import {
|
||||
createPresignedPutUrl,
|
||||
createS3Client,
|
||||
headStoredObject,
|
||||
loadStorageConfig,
|
||||
worldUploadObjectKey,
|
||||
} from '@hexahost/storage';
|
||||
|
||||
import { AuditService } from '../../auth/audit.service';
|
||||
import { NodeBridgeService } from '../../nodes/node-bridge.service';
|
||||
import { PrismaService } from '../../prisma/prisma.service';
|
||||
import { findOwnedServer } from '../shared/server-ownership.util';
|
||||
import { ServerStateService } from '../server-state.service';
|
||||
import { SERVER_BACKUPS_QUEUE } from '../backups/backups.service';
|
||||
|
||||
const WORLD_MANAGE_STATUSES = new Set<GameServer['status']>(['STOPPED']);
|
||||
|
||||
@Injectable()
|
||||
export class WorldsService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly nodeBridge: NodeBridgeService,
|
||||
private readonly serverState: ServerStateService,
|
||||
private readonly auditService: AuditService,
|
||||
@Inject(SERVER_BACKUPS_QUEUE) private readonly backupsQueue: Queue,
|
||||
) {}
|
||||
|
||||
async getWorldInfo(userId: string, serverId: string): Promise<WorldInfoResponse> {
|
||||
const server = await this.requireNode(serverId, userId);
|
||||
|
||||
const response = await this.nodeBridge.sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.world.validate',
|
||||
{
|
||||
serverId,
|
||||
generation: server.version,
|
||||
worldName: server.activeWorldName,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.success) {
|
||||
throw new ServiceUnavailableException(
|
||||
response.errorMessage ?? 'Failed to load world info',
|
||||
);
|
||||
}
|
||||
|
||||
const result = response.result as {
|
||||
worlds?: Array<{
|
||||
name: string;
|
||||
path: string;
|
||||
sizeBytes: number;
|
||||
hasLevelDat: boolean;
|
||||
}>;
|
||||
};
|
||||
|
||||
return {
|
||||
activeWorldName: server.activeWorldName,
|
||||
worlds: (result.worlds ?? []).map((world) => ({
|
||||
name: world.name,
|
||||
path: world.path,
|
||||
sizeBytes: String(world.sizeBytes),
|
||||
hasLevelDat: world.hasLevelDat,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async initiateUpload(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
input: InitiateWorldUpload,
|
||||
): Promise<WorldUploadInitResponse> {
|
||||
const server = await this.requireStoppedServer(userId, serverId);
|
||||
const uploadId = randomUUID();
|
||||
const config = loadStorageConfig();
|
||||
const client = createS3Client(config);
|
||||
const s3Key = worldUploadObjectKey(serverId, uploadId);
|
||||
const expiresAt = new Date(Date.now() + 60 * 60 * 1000);
|
||||
const uploadUrl = await createPresignedPutUrl(
|
||||
client,
|
||||
config.S3_BUCKET,
|
||||
s3Key,
|
||||
60 * 60,
|
||||
'application/gzip',
|
||||
);
|
||||
|
||||
await this.prisma.worldUpload.create({
|
||||
data: {
|
||||
id: uploadId,
|
||||
serverId,
|
||||
status: 'UPLOADING',
|
||||
s3Key,
|
||||
worldName: input.worldName,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
uploadId,
|
||||
uploadUrl,
|
||||
expiresAt: expiresAt.toISOString(),
|
||||
s3Key,
|
||||
};
|
||||
}
|
||||
|
||||
async completeUpload(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
uploadId: string,
|
||||
): Promise<WorldUploadResponse> {
|
||||
const server = await this.requireStoppedServer(userId, serverId);
|
||||
const upload = await this.prisma.worldUpload.findFirstOrThrow({
|
||||
where: { id: uploadId, serverId },
|
||||
});
|
||||
|
||||
const config = loadStorageConfig();
|
||||
const client = createS3Client(config);
|
||||
const head = await headStoredObject(client, config.S3_BUCKET, upload.s3Key);
|
||||
|
||||
if (head.sizeBytes <= 0) {
|
||||
throw new BadRequestException('Uploaded world archive is empty');
|
||||
}
|
||||
|
||||
const updated = await this.prisma.worldUpload.update({
|
||||
where: { id: uploadId },
|
||||
data: {
|
||||
status: 'READY',
|
||||
sizeBytes: BigInt(head.sizeBytes),
|
||||
},
|
||||
});
|
||||
|
||||
await this.backupsQueue.add('apply-world-upload', {
|
||||
uploadId,
|
||||
serverId,
|
||||
});
|
||||
|
||||
await this.auditService.record({
|
||||
action: 'server.world.upload',
|
||||
userId,
|
||||
entityType: 'game_server',
|
||||
entityId: serverId,
|
||||
metadata: { uploadId, worldName: upload.worldName },
|
||||
});
|
||||
|
||||
return this.toUploadResponse(updated);
|
||||
}
|
||||
|
||||
async requestWorldDownload(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
): Promise<WorldDownloadUrlResponse> {
|
||||
const server = await this.requireNode(serverId, userId);
|
||||
|
||||
if (server.status !== 'RUNNING' && server.status !== 'STOPPED') {
|
||||
throw new BadRequestException(
|
||||
`Cannot download world while server is ${server.status}`,
|
||||
);
|
||||
}
|
||||
|
||||
const backup = await this.prisma.serverBackup.create({
|
||||
data: {
|
||||
serverId,
|
||||
type: 'MANUAL',
|
||||
status: 'PENDING',
|
||||
label: 'World download',
|
||||
},
|
||||
});
|
||||
|
||||
const previousStatus = server.status as 'RUNNING' | 'STOPPED';
|
||||
|
||||
await this.prisma.gameServer.update({
|
||||
where: { id: serverId },
|
||||
data: { status: 'BACKING_UP' },
|
||||
});
|
||||
|
||||
await this.prisma.gameServerStateTransition.create({
|
||||
data: {
|
||||
gameServerId: serverId,
|
||||
fromStatus: server.status,
|
||||
toStatus: 'BACKING_UP',
|
||||
reason: 'World download backup started',
|
||||
actorId: userId,
|
||||
},
|
||||
});
|
||||
|
||||
await this.backupsQueue.add('create-backup', {
|
||||
backupId: backup.id,
|
||||
serverId,
|
||||
previousStatus,
|
||||
});
|
||||
|
||||
return {
|
||||
backupId: backup.id,
|
||||
downloadUrl: '',
|
||||
expiresAt: new Date(Date.now() + 15 * 60 * 1000).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
private async requireNode(serverId: string, userId: string): Promise<GameServer> {
|
||||
const server = await findOwnedServer(this.prisma, userId, serverId);
|
||||
if (!server.nodeId) {
|
||||
throw new BadRequestException('Server is not assigned to a node');
|
||||
}
|
||||
return server;
|
||||
}
|
||||
|
||||
private async requireStoppedServer(
|
||||
userId: string,
|
||||
serverId: string,
|
||||
): Promise<GameServer> {
|
||||
const server = await this.requireNode(serverId, userId);
|
||||
this.serverState.assertNotInProgress(server.status);
|
||||
|
||||
if (!WORLD_MANAGE_STATUSES.has(server.status)) {
|
||||
throw new BadRequestException(
|
||||
`World changes require server to be STOPPED (current: ${server.status})`,
|
||||
);
|
||||
}
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
private toUploadResponse(upload: WorldUpload): WorldUploadResponse {
|
||||
return {
|
||||
id: upload.id,
|
||||
serverId: upload.serverId,
|
||||
status: upload.status,
|
||||
worldName: upload.worldName,
|
||||
sizeBytes: upload.sizeBytes?.toString() ?? null,
|
||||
failureReason: upload.failureReason,
|
||||
createdAt: upload.createdAt.toISOString(),
|
||||
completedAt: upload.completedAt?.toISOString() ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
159
apps/node-agent/internal/archive/archive.go
Normal file
159
apps/node-agent/internal/archive/archive.go
Normal file
@@ -0,0 +1,159 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CreateTarGz archives sourceDir into outputPath and returns checksum and size.
|
||||
func CreateTarGz(sourceDir, outputPath string) (sha256sum string, size int64, err error) {
|
||||
sourceDir = filepath.Clean(sourceDir)
|
||||
|
||||
info, err := os.Stat(sourceDir)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", 0, fmt.Errorf("source is not a directory")
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
file, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
writer := io.MultiWriter(file, hasher)
|
||||
gzipWriter := gzip.NewWriter(writer)
|
||||
tarWriter := tar.NewWriter(gzipWriter)
|
||||
|
||||
err = filepath.Walk(sourceDir, func(path string, entry os.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(sourceDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
rel = filepath.ToSlash(rel)
|
||||
header, err := tar.FileInfoHeader(entry, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = rel
|
||||
|
||||
if err := tarWriter.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
sourceFile, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
_, err = io.Copy(tarWriter, sourceFile)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
if err := tarWriter.Close(); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hasher.Sum(nil)), stat.Size(), nil
|
||||
}
|
||||
|
||||
// ExtractTarGz extracts archivePath into destDir.
|
||||
func ExtractTarGz(archivePath, destDir string) error {
|
||||
destDir = filepath.Clean(destDir)
|
||||
if err := os.MkdirAll(destDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
target := filepath.Join(destDir, filepath.FromSlash(header.Name))
|
||||
if !strings.HasPrefix(filepath.Clean(target), destDir+string(os.PathSeparator)) && filepath.Clean(target) != destDir {
|
||||
return fmt.Errorf("archive entry escapes destination: %s", header.Name)
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(out, tarReader); err != nil {
|
||||
out.Close()
|
||||
return err
|
||||
}
|
||||
out.Close()
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -40,6 +40,10 @@ const (
|
||||
TypeServerFilesWrite MessageType = "server.files.write"
|
||||
TypeServerFilesUploadDone MessageType = "server.files.upload.complete"
|
||||
TypeServerWorldValidate MessageType = "server.world.validate"
|
||||
TypeServerWorldArchive MessageType = "server.world.archive"
|
||||
TypeServerWorldReplace MessageType = "server.world.replace"
|
||||
TypeServerStorageUpload MessageType = "server.storage.upload"
|
||||
TypeServerStorageDownload MessageType = "server.storage.download"
|
||||
TypeServerMetricsSub MessageType = "server.metrics.subscribe"
|
||||
TypeServerLogsSubscribe MessageType = "server.logs.subscribe"
|
||||
)
|
||||
@@ -72,6 +76,10 @@ var controlToAgentTypes = map[MessageType]struct{}{
|
||||
TypeServerFilesWrite: {},
|
||||
TypeServerFilesUploadDone: {},
|
||||
TypeServerWorldValidate: {},
|
||||
TypeServerWorldArchive: {},
|
||||
TypeServerWorldReplace: {},
|
||||
TypeServerStorageUpload: {},
|
||||
TypeServerStorageDownload: {},
|
||||
TypeServerMetricsSub: {},
|
||||
TypeServerLogsSubscribe: {},
|
||||
}
|
||||
@@ -178,6 +186,40 @@ type ServerFilesWritePayload struct {
|
||||
Create bool `json:"create"`
|
||||
}
|
||||
|
||||
// ServerWorldValidatePayload validates a Minecraft world directory.
|
||||
type ServerWorldValidatePayload struct {
|
||||
OperationMeta
|
||||
WorldName string `json:"worldName"`
|
||||
}
|
||||
|
||||
// ServerWorldArchivePayload archives a world directory.
|
||||
type ServerWorldArchivePayload struct {
|
||||
OperationMeta
|
||||
WorldName string `json:"worldName"`
|
||||
ArchiveID string `json:"archiveId"`
|
||||
}
|
||||
|
||||
// ServerStorageUploadPayload uploads a local file to a presigned URL.
|
||||
type ServerStorageUploadPayload struct {
|
||||
OperationMeta
|
||||
LocalPath string `json:"localPath"`
|
||||
UploadURL string `json:"uploadUrl"`
|
||||
}
|
||||
|
||||
// ServerStorageDownloadPayload downloads a remote file to a local path.
|
||||
type ServerStorageDownloadPayload struct {
|
||||
OperationMeta
|
||||
DownloadURL string `json:"downloadUrl"`
|
||||
LocalPath string `json:"localPath"`
|
||||
}
|
||||
|
||||
// ServerWorldReplacePayload replaces a world from a local archive.
|
||||
type ServerWorldReplacePayload struct {
|
||||
OperationMeta
|
||||
WorldName string `json:"worldName"`
|
||||
ArchivePath string `json:"archivePath"`
|
||||
}
|
||||
|
||||
// FileEntry describes a file or directory in a list response.
|
||||
type FileEntry struct {
|
||||
Name string `json:"name"`
|
||||
|
||||
@@ -5,11 +5,16 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/archive"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/docker"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/files"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/protocol"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/transfer"
|
||||
worldpkg "github.com/hexahost/gamecloud/node-agent/internal/world"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -352,6 +357,219 @@ func (m *Manager) HandleFilesWrite(ctx context.Context, correlationID string, pa
|
||||
})
|
||||
}
|
||||
|
||||
// HandleBackupPrepare runs save-off and save-all flush via RCON.
|
||||
func (m *Manager) HandleBackupPrepare(ctx context.Context, correlationID string, meta protocol.OperationMeta) {
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.failOperation(ctx, correlationID, meta, "backup.prepare", "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
if rec.State != ServerStateRunning {
|
||||
m.completeOperation(ctx, correlationID, meta, "backup.prepare", map[string]string{"skipped": "not_running"})
|
||||
return
|
||||
}
|
||||
|
||||
for _, command := range []string{"save-off", "save-all flush"} {
|
||||
if _, _, err := m.docker.ExecRcon(ctx, rec.ContainerID, command); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "backup.prepare", "BACKUP_PREPARE_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "backup.prepare", map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// HandleBackupRelease runs save-on via RCON.
|
||||
func (m *Manager) HandleBackupRelease(ctx context.Context, correlationID string, meta protocol.OperationMeta) {
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.failOperation(ctx, correlationID, meta, "backup.release", "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
if rec.State != ServerStateRunning {
|
||||
m.completeOperation(ctx, correlationID, meta, "backup.release", map[string]string{"skipped": "not_running"})
|
||||
return
|
||||
}
|
||||
|
||||
if _, _, err := m.docker.ExecRcon(ctx, rec.ContainerID, "save-on"); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "backup.release", "BACKUP_RELEASE_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "backup.release", map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// HandleWorldValidate validates a world directory and lists worlds.
|
||||
func (m *Manager) HandleWorldValidate(ctx context.Context, correlationID string, payload protocol.ServerWorldValidatePayload) {
|
||||
meta := payload.OperationMeta
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.failOperation(ctx, correlationID, meta, "world.validate", "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
worldName := payload.WorldName
|
||||
if worldName == "" {
|
||||
worldName = "world"
|
||||
}
|
||||
|
||||
if err := worldpkg.ValidateWorld(rec.DataPath, worldName); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.validate", "WORLD_INVALID", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
worlds, err := worldpkg.ListWorlds(rec.DataPath, worldName)
|
||||
if err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.validate", "WORLD_LIST_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "world.validate", map[string]any{
|
||||
"valid": true,
|
||||
"worldName": worldName,
|
||||
"worlds": worlds,
|
||||
})
|
||||
}
|
||||
|
||||
// HandleWorldArchive creates a tar.gz archive of a world directory.
|
||||
func (m *Manager) HandleWorldArchive(ctx context.Context, correlationID string, payload protocol.ServerWorldArchivePayload) {
|
||||
meta := payload.OperationMeta
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.failOperation(ctx, correlationID, meta, "world.archive", "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
worldName := payload.WorldName
|
||||
if worldName == "" {
|
||||
worldName = "world"
|
||||
}
|
||||
|
||||
if err := worldpkg.ValidateWorld(rec.DataPath, worldName); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.archive", "WORLD_INVALID", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
archiveDir := filepath.Join(rec.DataPath, ".hgc", "archives")
|
||||
archiveID := payload.ArchiveID
|
||||
if archiveID == "" {
|
||||
archiveID = meta.ServerID
|
||||
}
|
||||
outputPath := filepath.Join(archiveDir, archiveID+".tar.gz")
|
||||
sourceDir := filepath.Join(rec.DataPath, worldName)
|
||||
|
||||
checksum, size, err := archive.CreateTarGz(sourceDir, outputPath)
|
||||
if err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.archive", "ARCHIVE_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "world.archive", map[string]any{
|
||||
"localPath": outputPath,
|
||||
"sha256": checksum,
|
||||
"sizeBytes": size,
|
||||
"worldName": worldName,
|
||||
})
|
||||
}
|
||||
|
||||
// HandleStorageUpload uploads a local archive to object storage.
|
||||
func (m *Manager) HandleStorageUpload(ctx context.Context, correlationID string, payload protocol.ServerStorageUploadPayload) {
|
||||
meta := payload.OperationMeta
|
||||
|
||||
if err := transfer.UploadFile(payload.LocalPath, payload.UploadURL); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "storage.upload", "UPLOAD_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "storage.upload", map[string]string{
|
||||
"localPath": payload.LocalPath,
|
||||
})
|
||||
}
|
||||
|
||||
// HandleStorageDownload downloads an archive from object storage.
|
||||
func (m *Manager) HandleStorageDownload(ctx context.Context, correlationID string, payload protocol.ServerStorageDownloadPayload) {
|
||||
meta := payload.OperationMeta
|
||||
|
||||
if err := transfer.DownloadFile(payload.DownloadURL, payload.LocalPath); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "storage.download", "DOWNLOAD_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "storage.download", map[string]string{
|
||||
"localPath": payload.LocalPath,
|
||||
})
|
||||
}
|
||||
|
||||
// HandleWorldReplace replaces a world directory from a local archive.
|
||||
func (m *Manager) HandleWorldReplace(ctx context.Context, correlationID string, payload protocol.ServerWorldReplacePayload) {
|
||||
meta := payload.OperationMeta
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
if rec.State == ServerStateRunning {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "SERVER_RUNNING", "server must be stopped before world replace")
|
||||
return
|
||||
}
|
||||
|
||||
worldName := payload.WorldName
|
||||
if worldName == "" {
|
||||
worldName = "world"
|
||||
}
|
||||
|
||||
worldDir := filepath.Join(rec.DataPath, worldName)
|
||||
tempDir := filepath.Join(rec.DataPath, ".hgc", "restore", meta.ServerID)
|
||||
if err := os.RemoveAll(tempDir); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "RESTORE_PREP_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(tempDir, 0o755); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "RESTORE_PREP_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := archive.ExtractTarGz(payload.ArchivePath, tempDir); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "EXTRACT_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
extractedWorld := filepath.Join(tempDir, worldName)
|
||||
if info, err := os.Stat(extractedWorld); err != nil || !info.IsDir() {
|
||||
extractedWorld = tempDir
|
||||
}
|
||||
|
||||
if err := worldpkg.ValidateWorld(filepath.Dir(extractedWorld), filepath.Base(extractedWorld)); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "WORLD_INVALID", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
backupDir := worldDir + ".bak"
|
||||
_ = os.RemoveAll(backupDir)
|
||||
if _, err := os.Stat(worldDir); err == nil {
|
||||
if err := os.Rename(worldDir, backupDir); err != nil {
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "REPLACE_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Rename(extractedWorld, worldDir); err != nil {
|
||||
_ = os.Rename(backupDir, worldDir)
|
||||
m.failOperation(ctx, correlationID, meta, "world.replace", "REPLACE_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
_ = os.RemoveAll(tempDir)
|
||||
_ = os.RemoveAll(backupDir)
|
||||
|
||||
m.completeOperation(ctx, correlationID, meta, "world.replace", map[string]string{
|
||||
"worldName": worldName,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Manager) getServer(serverID string) (*serverRecord, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
73
apps/node-agent/internal/transfer/transfer.go
Normal file
73
apps/node-agent/internal/transfer/transfer.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package transfer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// UploadFile PUTs a local file to a presigned URL.
|
||||
func UploadFile(localPath, uploadURL string) error {
|
||||
file, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
request, err := http.NewRequest(http.MethodPut, uploadURL, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.ContentLength = stat.Size()
|
||||
request.Header.Set("Content-Type", "application/gzip")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Minute}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return fmt.Errorf("upload failed with status %d: %s", response.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DownloadFile GETs a remote file to a local path.
|
||||
func DownloadFile(downloadURL, localPath string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Minute}
|
||||
response, err := client.Get(downloadURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return fmt.Errorf("download failed with status %d: %s", response.StatusCode, string(body))
|
||||
}
|
||||
|
||||
file, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(file, response.Body)
|
||||
return err
|
||||
}
|
||||
114
apps/node-agent/internal/world/world.go
Normal file
114
apps/node-agent/internal/world/world.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package world
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Summary describes a discovered Minecraft world directory.
|
||||
type Summary struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
SizeBytes int64 `json:"sizeBytes"`
|
||||
HasLevelDat bool `json:"hasLevelDat"`
|
||||
}
|
||||
|
||||
// ValidateWorld checks that a world directory contains level.dat.
|
||||
func ValidateWorld(dataPath, worldName string) error {
|
||||
worldName = strings.TrimSpace(worldName)
|
||||
if worldName == "" || strings.Contains(worldName, "..") || strings.ContainsAny(worldName, `/\`) {
|
||||
return fmt.Errorf("invalid world name")
|
||||
}
|
||||
|
||||
worldDir := filepath.Join(dataPath, worldName)
|
||||
levelDat := filepath.Join(worldDir, "level.dat")
|
||||
|
||||
info, err := os.Stat(levelDat)
|
||||
if err != nil {
|
||||
return fmt.Errorf("level.dat not found in world directory")
|
||||
}
|
||||
if info.IsDir() {
|
||||
return fmt.Errorf("level.dat is not a file")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListWorlds scans the server data directory for world folders.
|
||||
func ListWorlds(dataPath, activeWorldName string) ([]Summary, error) {
|
||||
entries, err := os.ReadDir(dataPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
worlds := make([]Summary, 0)
|
||||
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
worldPath := filepath.Join(dataPath, entry.Name())
|
||||
levelDat := filepath.Join(worldPath, "level.dat")
|
||||
hasLevelDat := false
|
||||
if info, err := os.Stat(levelDat); err == nil && !info.IsDir() {
|
||||
hasLevelDat = true
|
||||
}
|
||||
|
||||
size, err := dirSize(worldPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
worlds = append(worlds, Summary{
|
||||
Name: entry.Name(),
|
||||
Path: entry.Name(),
|
||||
SizeBytes: size,
|
||||
HasLevelDat: hasLevelDat,
|
||||
})
|
||||
}
|
||||
|
||||
if activeWorldName != "" {
|
||||
found := false
|
||||
for _, world := range worlds {
|
||||
if world.Name == activeWorldName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
if err := ValidateWorld(dataPath, activeWorldName); err == nil {
|
||||
size, sizeErr := dirSize(filepath.Join(dataPath, activeWorldName))
|
||||
if sizeErr != nil {
|
||||
return nil, sizeErr
|
||||
}
|
||||
worlds = append(worlds, Summary{
|
||||
Name: activeWorldName,
|
||||
Path: activeWorldName,
|
||||
SizeBytes: size,
|
||||
HasLevelDat: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return worlds, nil
|
||||
}
|
||||
|
||||
func dirSize(root string) (int64, error) {
|
||||
var total int64
|
||||
|
||||
err := filepath.Walk(root, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return total, err
|
||||
}
|
||||
@@ -314,6 +314,62 @@ func (c *Client) dispatch(ctx context.Context, env *protocol.Envelope) {
|
||||
}
|
||||
c.runtime.HandleFilesWrite(ctx, env.MessageID, payload)
|
||||
|
||||
case protocol.TypeServerBackupPrepare:
|
||||
var meta protocol.OperationMeta
|
||||
if err := json.Unmarshal(env.Payload, &meta); err != nil {
|
||||
c.log.Warn("decode server.backup.prepare", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleBackupPrepare(ctx, env.MessageID, meta)
|
||||
|
||||
case protocol.TypeServerBackupRelease:
|
||||
var meta protocol.OperationMeta
|
||||
if err := json.Unmarshal(env.Payload, &meta); err != nil {
|
||||
c.log.Warn("decode server.backup.release", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleBackupRelease(ctx, env.MessageID, meta)
|
||||
|
||||
case protocol.TypeServerWorldValidate:
|
||||
var payload protocol.ServerWorldValidatePayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.world.validate", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleWorldValidate(ctx, env.MessageID, payload)
|
||||
|
||||
case protocol.TypeServerWorldArchive:
|
||||
var payload protocol.ServerWorldArchivePayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.world.archive", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleWorldArchive(ctx, env.MessageID, payload)
|
||||
|
||||
case protocol.TypeServerStorageUpload:
|
||||
var payload protocol.ServerStorageUploadPayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.storage.upload", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleStorageUpload(ctx, env.MessageID, payload)
|
||||
|
||||
case protocol.TypeServerStorageDownload:
|
||||
var payload protocol.ServerStorageDownloadPayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.storage.download", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleStorageDownload(ctx, env.MessageID, payload)
|
||||
|
||||
case protocol.TypeServerWorldReplace:
|
||||
var payload protocol.ServerWorldReplacePayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.world.replace", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleWorldReplace(ctx, env.MessageID, payload)
|
||||
|
||||
default:
|
||||
c.log.Debug("ignored control message", "type", env.Type)
|
||||
}
|
||||
|
||||
@@ -190,7 +190,9 @@
|
||||
"console": "Konsole",
|
||||
"files": "Dateien",
|
||||
"properties": "Eigenschaften",
|
||||
"players": "Spieler"
|
||||
"players": "Spieler",
|
||||
"worlds": "Welten",
|
||||
"backups": "Backups"
|
||||
},
|
||||
"console": {
|
||||
"loading": "Wird geladen…",
|
||||
@@ -274,6 +276,47 @@
|
||||
"errors": {
|
||||
"loadFailed": "Spielerliste konnte nicht geladen werden."
|
||||
}
|
||||
},
|
||||
"worlds": {
|
||||
"loading": "Wird geladen…",
|
||||
"loadError": "Weltinformationen konnten nicht geladen werden.",
|
||||
"activeWorld": "Aktive Welt: {name}",
|
||||
"stoppedHint": "Upload und Ersetzen erfordern einen gestoppten Server.",
|
||||
"worldName": "Weltname",
|
||||
"uploadArchive": "Welt-Archiv (.tar.gz)",
|
||||
"uploadError": "Welt-Upload fehlgeschlagen.",
|
||||
"download": "Welt herunterladen",
|
||||
"downloadError": "Download konnte nicht gestartet werden.",
|
||||
"downloadPending": "Backup wird erstellt. Versuchen Sie den Download in Kürze erneut.",
|
||||
"empty": "Keine Welten gefunden.",
|
||||
"validWorld": "Gültige Welt",
|
||||
"invalidWorld": "Ungültig",
|
||||
"active": "Aktiv"
|
||||
},
|
||||
"backups": {
|
||||
"loading": "Wird geladen…",
|
||||
"loadError": "Backups konnten nicht geladen werden.",
|
||||
"create": "Backup erstellen",
|
||||
"manualLabel": "Manuelles Backup",
|
||||
"createError": "Backup konnte nicht erstellt werden.",
|
||||
"download": "Download",
|
||||
"downloadError": "Download-URL konnte nicht erstellt werden.",
|
||||
"restore": "Wiederherstellen",
|
||||
"restoreConfirm": "Welt wirklich aus diesem Backup wiederherstellen? Der Server muss gestoppt sein.",
|
||||
"restoreError": "Wiederherstellung fehlgeschlagen.",
|
||||
"delete": "Löschen",
|
||||
"deleteConfirm": "Backup wirklich löschen?",
|
||||
"deleteError": "Backup konnte nicht gelöscht werden.",
|
||||
"enableSchedule": "Geplante Backups aktivieren",
|
||||
"disableSchedule": "Geplante Backups deaktivieren",
|
||||
"scheduleError": "Zeitplan konnte nicht aktualisiert werden.",
|
||||
"scheduleInfo": "Intervall: alle {hours}h · Aufbewahrung: {retention} Backups",
|
||||
"label": "Bezeichnung",
|
||||
"status": "Status",
|
||||
"size": "Größe",
|
||||
"created": "Erstellt",
|
||||
"actions": "Aktionen",
|
||||
"empty": "Noch keine Backups vorhanden."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,7 +190,9 @@
|
||||
"console": "Console",
|
||||
"files": "Files",
|
||||
"properties": "Properties",
|
||||
"players": "Players"
|
||||
"players": "Players",
|
||||
"worlds": "Worlds",
|
||||
"backups": "Backups"
|
||||
},
|
||||
"console": {
|
||||
"loading": "Loading…",
|
||||
@@ -274,6 +276,47 @@
|
||||
"errors": {
|
||||
"loadFailed": "Could not load player list."
|
||||
}
|
||||
},
|
||||
"worlds": {
|
||||
"loading": "Loading…",
|
||||
"loadError": "Could not load world information.",
|
||||
"activeWorld": "Active world: {name}",
|
||||
"stoppedHint": "Upload and replace require a stopped server.",
|
||||
"worldName": "World name",
|
||||
"uploadArchive": "World archive (.tar.gz)",
|
||||
"uploadError": "World upload failed.",
|
||||
"download": "Download world",
|
||||
"downloadError": "Could not start download.",
|
||||
"downloadPending": "Backup is being created. Try the download again shortly.",
|
||||
"empty": "No worlds found.",
|
||||
"validWorld": "Valid world",
|
||||
"invalidWorld": "Invalid",
|
||||
"active": "Active"
|
||||
},
|
||||
"backups": {
|
||||
"loading": "Loading…",
|
||||
"loadError": "Could not load backups.",
|
||||
"create": "Create backup",
|
||||
"manualLabel": "Manual backup",
|
||||
"createError": "Could not create backup.",
|
||||
"download": "Download",
|
||||
"downloadError": "Could not create download URL.",
|
||||
"restore": "Restore",
|
||||
"restoreConfirm": "Restore the world from this backup? The server must be stopped.",
|
||||
"restoreError": "Restore failed.",
|
||||
"delete": "Delete",
|
||||
"deleteConfirm": "Delete this backup?",
|
||||
"deleteError": "Could not delete backup.",
|
||||
"enableSchedule": "Enable scheduled backups",
|
||||
"disableSchedule": "Disable scheduled backups",
|
||||
"scheduleError": "Could not update schedule.",
|
||||
"scheduleInfo": "Interval: every {hours}h · Retention: {retention} backups",
|
||||
"label": "Label",
|
||||
"status": "Status",
|
||||
"size": "Size",
|
||||
"created": "Created",
|
||||
"actions": "Actions",
|
||||
"empty": "No backups yet."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
13
apps/web/src/app/[locale]/servers/[id]/backups/page.tsx
Normal file
13
apps/web/src/app/[locale]/servers/[id]/backups/page.tsx
Normal file
@@ -0,0 +1,13 @@
|
||||
import { setRequestLocale } from "next-intl/server";
|
||||
import { ServerBackups } from "@/components/servers/server-backups";
|
||||
|
||||
interface ServerBackupsPageProps {
|
||||
params: Promise<{ locale: string; id: string }>;
|
||||
}
|
||||
|
||||
export default async function ServerBackupsPage({ params }: ServerBackupsPageProps) {
|
||||
const { locale, id } = await params;
|
||||
setRequestLocale(locale);
|
||||
|
||||
return <ServerBackups serverId={id} />;
|
||||
}
|
||||
13
apps/web/src/app/[locale]/servers/[id]/worlds/page.tsx
Normal file
13
apps/web/src/app/[locale]/servers/[id]/worlds/page.tsx
Normal file
@@ -0,0 +1,13 @@
|
||||
import { setRequestLocale } from "next-intl/server";
|
||||
import { ServerWorlds } from "@/components/servers/server-worlds";
|
||||
|
||||
interface ServerWorldsPageProps {
|
||||
params: Promise<{ locale: string; id: string }>;
|
||||
}
|
||||
|
||||
export default async function ServerWorldsPage({ params }: ServerWorldsPageProps) {
|
||||
const { locale, id } = await params;
|
||||
setRequestLocale(locale);
|
||||
|
||||
return <ServerWorlds serverId={id} />;
|
||||
}
|
||||
237
apps/web/src/components/servers/server-backups.tsx
Normal file
237
apps/web/src/components/servers/server-backups.tsx
Normal file
@@ -0,0 +1,237 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
import { Button } from "@hexahost/ui";
|
||||
|
||||
import {
|
||||
createBackup,
|
||||
deleteBackup,
|
||||
getBackupDownloadUrl,
|
||||
getBackupSchedule,
|
||||
listBackups,
|
||||
restoreBackup,
|
||||
updateBackupSchedule,
|
||||
type BackupItem,
|
||||
type BackupSchedule,
|
||||
} from "@/lib/api/backups";
|
||||
|
||||
interface ServerBackupsProps {
|
||||
serverId: string;
|
||||
}
|
||||
|
||||
function formatBytes(value: string | null): string {
|
||||
if (!value) {
|
||||
return "—";
|
||||
}
|
||||
const bytes = Number(value);
|
||||
if (Number.isNaN(bytes)) {
|
||||
return value;
|
||||
}
|
||||
if (bytes < 1024) {
|
||||
return `${bytes} B`;
|
||||
}
|
||||
if (bytes < 1024 * 1024) {
|
||||
return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
}
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
export function ServerBackups({ serverId }: ServerBackupsProps) {
|
||||
const t = useTranslations("servers.backups");
|
||||
const [backups, setBackups] = useState<BackupItem[]>([]);
|
||||
const [schedule, setSchedule] = useState<BackupSchedule | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const [backupList, scheduleData] = await Promise.all([
|
||||
listBackups(serverId),
|
||||
getBackupSchedule(serverId),
|
||||
]);
|
||||
setBackups(backupList.backups);
|
||||
setSchedule(scheduleData);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("loadError"));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [serverId, t]);
|
||||
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
async function handleCreateBackup() {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
await createBackup(serverId, t("manualLabel"));
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("createError"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDownload(backupId: string) {
|
||||
try {
|
||||
const { downloadUrl } = await getBackupDownloadUrl(serverId, backupId);
|
||||
window.open(downloadUrl, "_blank", "noopener,noreferrer");
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("downloadError"));
|
||||
}
|
||||
}
|
||||
|
||||
async function handleRestore(backupId: string) {
|
||||
if (!window.confirm(t("restoreConfirm"))) {
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
try {
|
||||
await restoreBackup(serverId, backupId);
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("restoreError"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete(backupId: string) {
|
||||
if (!window.confirm(t("deleteConfirm"))) {
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
try {
|
||||
await deleteBackup(serverId, backupId);
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("deleteError"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleSchedule(enabled: boolean) {
|
||||
if (!schedule) {
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
try {
|
||||
const updated = await updateBackupSchedule(serverId, { enabled });
|
||||
setSchedule(updated);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("scheduleError"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return <p className="text-sm text-zinc-500">{t("loading")}</p>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{error ? (
|
||||
<p className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-sm text-red-700 dark:border-red-900 dark:bg-red-950 dark:text-red-300">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<div className="flex flex-wrap items-center gap-3">
|
||||
<Button onClick={() => void handleCreateBackup()} disabled={busy}>
|
||||
{t("create")}
|
||||
</Button>
|
||||
{schedule ? (
|
||||
<Button
|
||||
variant="secondary"
|
||||
onClick={() => void toggleSchedule(!schedule.enabled)}
|
||||
disabled={busy}
|
||||
>
|
||||
{schedule.enabled ? t("disableSchedule") : t("enableSchedule")}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{schedule ? (
|
||||
<p className="text-sm text-zinc-600 dark:text-zinc-400">
|
||||
{t("scheduleInfo", {
|
||||
hours: schedule.intervalHours,
|
||||
retention: schedule.retentionCount,
|
||||
})}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<div className="overflow-x-auto rounded-lg border border-zinc-200 dark:border-zinc-800">
|
||||
<table className="min-w-full divide-y divide-zinc-200 text-sm dark:divide-zinc-800">
|
||||
<thead className="bg-zinc-50 dark:bg-zinc-900">
|
||||
<tr>
|
||||
<th className="px-4 py-2 text-left font-medium">{t("label")}</th>
|
||||
<th className="px-4 py-2 text-left font-medium">{t("status")}</th>
|
||||
<th className="px-4 py-2 text-left font-medium">{t("size")}</th>
|
||||
<th className="px-4 py-2 text-left font-medium">{t("created")}</th>
|
||||
<th className="px-4 py-2 text-right font-medium">{t("actions")}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-zinc-200 dark:divide-zinc-800">
|
||||
{backups.length === 0 ? (
|
||||
<tr>
|
||||
<td colSpan={5} className="px-4 py-6 text-center text-zinc-500">
|
||||
{t("empty")}
|
||||
</td>
|
||||
</tr>
|
||||
) : (
|
||||
backups.map((backup) => (
|
||||
<tr key={backup.id}>
|
||||
<td className="px-4 py-2">{backup.label ?? backup.type}</td>
|
||||
<td className="px-4 py-2">{backup.status}</td>
|
||||
<td className="px-4 py-2">{formatBytes(backup.sizeBytes)}</td>
|
||||
<td className="px-4 py-2">
|
||||
{new Date(backup.createdAt).toLocaleString()}
|
||||
</td>
|
||||
<td className="px-4 py-2 text-right space-x-2">
|
||||
{backup.status === "AVAILABLE" ? (
|
||||
<>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="secondary"
|
||||
onClick={() => void handleDownload(backup.id)}
|
||||
>
|
||||
{t("download")}
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
variant="secondary"
|
||||
onClick={() => void handleRestore(backup.id)}
|
||||
disabled={busy}
|
||||
>
|
||||
{t("restore")}
|
||||
</Button>
|
||||
</>
|
||||
) : null}
|
||||
<Button
|
||||
size="sm"
|
||||
variant="ghost"
|
||||
onClick={() => void handleDelete(backup.id)}
|
||||
disabled={busy}
|
||||
>
|
||||
{t("delete")}
|
||||
</Button>
|
||||
</td>
|
||||
</tr>
|
||||
))
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -13,6 +13,8 @@ const tabs = [
|
||||
{ key: "files", suffix: "/files" },
|
||||
{ key: "properties", suffix: "/properties" },
|
||||
{ key: "players", suffix: "/players" },
|
||||
{ key: "worlds", suffix: "/worlds" },
|
||||
{ key: "backups", suffix: "/backups" },
|
||||
] as const;
|
||||
|
||||
export function ServerTabs({ serverId }: ServerTabsProps) {
|
||||
|
||||
172
apps/web/src/components/servers/server-worlds.tsx
Normal file
172
apps/web/src/components/servers/server-worlds.tsx
Normal file
@@ -0,0 +1,172 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { useTranslations } from "next-intl";
|
||||
|
||||
import { Button } from "@hexahost/ui";
|
||||
|
||||
import {
|
||||
completeWorldUpload,
|
||||
getWorldInfo,
|
||||
initiateWorldUpload,
|
||||
requestWorldDownload,
|
||||
type WorldSummary,
|
||||
} from "@/lib/api/worlds";
|
||||
import { getBackupDownloadUrl } from "@/lib/api/backups";
|
||||
|
||||
interface ServerWorldsProps {
|
||||
serverId: string;
|
||||
}
|
||||
|
||||
function formatBytes(value: string): string {
|
||||
const bytes = Number(value);
|
||||
if (Number.isNaN(bytes)) {
|
||||
return value;
|
||||
}
|
||||
if (bytes < 1024 * 1024) {
|
||||
return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
}
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
|
||||
export function ServerWorlds({ serverId }: ServerWorldsProps) {
|
||||
const t = useTranslations("servers.worlds");
|
||||
const [activeWorldName, setActiveWorldName] = useState("world");
|
||||
const [worlds, setWorlds] = useState<WorldSummary[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [worldName, setWorldName] = useState("world");
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const info = await getWorldInfo(serverId);
|
||||
setActiveWorldName(info.activeWorldName);
|
||||
setWorlds(info.worlds);
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("loadError"));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [serverId, t]);
|
||||
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
async function handleUpload(file: File) {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
const init = await initiateWorldUpload(serverId, worldName);
|
||||
const response = await fetch(init.uploadUrl, {
|
||||
method: "PUT",
|
||||
body: file,
|
||||
headers: { "Content-Type": "application/gzip" },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(t("uploadError"));
|
||||
}
|
||||
await completeWorldUpload(serverId, init.uploadId);
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("uploadError"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDownload() {
|
||||
setBusy(true);
|
||||
setError(null);
|
||||
try {
|
||||
const { backupId } = await requestWorldDownload(serverId);
|
||||
const { downloadUrl } = await getBackupDownloadUrl(serverId, backupId);
|
||||
if (downloadUrl) {
|
||||
window.open(downloadUrl, "_blank", "noopener,noreferrer");
|
||||
} else {
|
||||
setError(t("downloadPending"));
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : t("downloadError"));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (loading) {
|
||||
return <p className="text-sm text-zinc-500">{t("loading")}</p>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
{error ? (
|
||||
<p className="rounded-md border border-red-200 bg-red-50 px-3 py-2 text-sm text-red-700 dark:border-red-900 dark:bg-red-950 dark:text-red-300">
|
||||
{error}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
<div className="rounded-lg border border-zinc-200 p-4 dark:border-zinc-800">
|
||||
<p className="text-sm text-zinc-600 dark:text-zinc-400">
|
||||
{t("activeWorld", { name: activeWorldName })}
|
||||
</p>
|
||||
<p className="mt-1 text-xs text-zinc-500">{t("stoppedHint")}</p>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap gap-3 items-end">
|
||||
<label className="block text-sm">
|
||||
<span className="mb-1 block text-zinc-600 dark:text-zinc-400">
|
||||
{t("worldName")}
|
||||
</span>
|
||||
<input
|
||||
className="rounded-md border border-zinc-300 bg-white px-3 py-2 text-sm dark:border-zinc-700 dark:bg-zinc-950"
|
||||
value={worldName}
|
||||
onChange={(event) => setWorldName(event.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label className="block text-sm">
|
||||
<span className="mb-1 block text-zinc-600 dark:text-zinc-400">
|
||||
{t("uploadArchive")}
|
||||
</span>
|
||||
<input
|
||||
type="file"
|
||||
accept=".tar.gz,.tgz,application/gzip"
|
||||
disabled={busy}
|
||||
onChange={(event) => {
|
||||
const file = event.target.files?.[0];
|
||||
if (file) {
|
||||
void handleUpload(file);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
<Button variant="secondary" onClick={() => void handleDownload()} disabled={busy}>
|
||||
{t("download")}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<ul className="divide-y divide-zinc-200 rounded-lg border border-zinc-200 dark:divide-zinc-800 dark:border-zinc-800">
|
||||
{worlds.length === 0 ? (
|
||||
<li className="px-4 py-6 text-center text-sm text-zinc-500">{t("empty")}</li>
|
||||
) : (
|
||||
worlds.map((world) => (
|
||||
<li key={world.name} className="flex items-center justify-between px-4 py-3 text-sm">
|
||||
<div>
|
||||
<p className="font-medium">{world.name}</p>
|
||||
<p className="text-zinc-500">
|
||||
{world.hasLevelDat ? t("validWorld") : t("invalidWorld")} ·{" "}
|
||||
{formatBytes(world.sizeBytes)}
|
||||
</p>
|
||||
</div>
|
||||
{world.name === activeWorldName ? (
|
||||
<span className="text-xs text-sky-600 dark:text-sky-400">{t("active")}</span>
|
||||
) : null}
|
||||
</li>
|
||||
))
|
||||
)}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
80
apps/web/src/lib/api/backups.ts
Normal file
80
apps/web/src/lib/api/backups.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { apiFetch } from "./auth";
|
||||
|
||||
export interface BackupItem {
|
||||
id: string;
|
||||
serverId: string;
|
||||
type: string;
|
||||
status: string;
|
||||
sizeBytes: string | null;
|
||||
sha256: string | null;
|
||||
label: string | null;
|
||||
failureReason: string | null;
|
||||
createdAt: string;
|
||||
completedAt: string | null;
|
||||
}
|
||||
|
||||
export interface BackupListResponse {
|
||||
backups: BackupItem[];
|
||||
}
|
||||
|
||||
export interface BackupSchedule {
|
||||
enabled: boolean;
|
||||
intervalHours: number;
|
||||
retentionCount: number;
|
||||
nextRunAt: string | null;
|
||||
lastRunAt: string | null;
|
||||
}
|
||||
|
||||
export async function listBackups(serverId: string): Promise<BackupListResponse> {
|
||||
return apiFetch<BackupListResponse>(`/servers/${serverId}/backups`);
|
||||
}
|
||||
|
||||
export async function createBackup(
|
||||
serverId: string,
|
||||
label?: string,
|
||||
): Promise<{ backup: BackupItem; jobId?: string }> {
|
||||
return apiFetch(`/servers/${serverId}/backups`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ label }),
|
||||
});
|
||||
}
|
||||
|
||||
export async function deleteBackup(
|
||||
serverId: string,
|
||||
backupId: string,
|
||||
): Promise<BackupItem> {
|
||||
return apiFetch(`/servers/${serverId}/backups/${backupId}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
}
|
||||
|
||||
export async function getBackupDownloadUrl(
|
||||
serverId: string,
|
||||
backupId: string,
|
||||
): Promise<{ downloadUrl: string; expiresAt: string }> {
|
||||
return apiFetch(`/servers/${serverId}/backups/${backupId}/download-url`);
|
||||
}
|
||||
|
||||
export async function restoreBackup(
|
||||
serverId: string,
|
||||
backupId: string,
|
||||
): Promise<unknown> {
|
||||
return apiFetch(`/servers/${serverId}/backups/${backupId}/restore`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ confirm: true }),
|
||||
});
|
||||
}
|
||||
|
||||
export async function getBackupSchedule(serverId: string): Promise<BackupSchedule> {
|
||||
return apiFetch(`/servers/${serverId}/backups/schedule`);
|
||||
}
|
||||
|
||||
export async function updateBackupSchedule(
|
||||
serverId: string,
|
||||
input: Partial<BackupSchedule>,
|
||||
): Promise<BackupSchedule> {
|
||||
return apiFetch(`/servers/${serverId}/backups/schedule`, {
|
||||
method: "PATCH",
|
||||
body: JSON.stringify(input),
|
||||
});
|
||||
}
|
||||
48
apps/web/src/lib/api/worlds.ts
Normal file
48
apps/web/src/lib/api/worlds.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
import { apiFetch } from "./auth";
|
||||
|
||||
export interface WorldSummary {
|
||||
name: string;
|
||||
path: string;
|
||||
sizeBytes: string;
|
||||
hasLevelDat: boolean;
|
||||
}
|
||||
|
||||
export interface WorldInfoResponse {
|
||||
activeWorldName: string;
|
||||
worlds: WorldSummary[];
|
||||
}
|
||||
|
||||
export async function getWorldInfo(serverId: string): Promise<WorldInfoResponse> {
|
||||
return apiFetch(`/servers/${serverId}/worlds`);
|
||||
}
|
||||
|
||||
export async function initiateWorldUpload(
|
||||
serverId: string,
|
||||
worldName: string,
|
||||
): Promise<{
|
||||
uploadId: string;
|
||||
uploadUrl: string;
|
||||
expiresAt: string;
|
||||
}> {
|
||||
return apiFetch(`/servers/${serverId}/worlds/uploads`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ worldName }),
|
||||
});
|
||||
}
|
||||
|
||||
export async function completeWorldUpload(
|
||||
serverId: string,
|
||||
uploadId: string,
|
||||
): Promise<unknown> {
|
||||
return apiFetch(`/servers/${serverId}/worlds/uploads/${uploadId}/complete`, {
|
||||
method: "POST",
|
||||
});
|
||||
}
|
||||
|
||||
export async function requestWorldDownload(
|
||||
serverId: string,
|
||||
): Promise<{ backupId: string }> {
|
||||
return apiFetch(`/servers/${serverId}/worlds/download`, {
|
||||
method: "POST",
|
||||
});
|
||||
}
|
||||
@@ -11,9 +11,10 @@
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "node --test test/**/*.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"dependencies": {
|
||||
"@hexahost/config": "workspace:*",
|
||||
"@hexahost/database": "workspace:*",
|
||||
"@hexahost/storage": "workspace:*",
|
||||
"bullmq": "^5.34.10",
|
||||
"ioredis": "^5.4.2",
|
||||
"nodemailer": "^6.10.0",
|
||||
|
||||
54
apps/worker/src/agent-bridge.ts
Normal file
54
apps/worker/src/agent-bridge.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { validateConfig } from '@hexahost/config';
|
||||
|
||||
import { logger } from './logger';
|
||||
import {
|
||||
publishNodeCommand,
|
||||
waitForNodeResponse,
|
||||
type NodeResponseMessage,
|
||||
} from './redis';
|
||||
|
||||
export interface AgentRequestResult extends NodeResponseMessage {
|
||||
result?: unknown;
|
||||
}
|
||||
|
||||
export async function sendAgentRequest(
|
||||
nodeId: string,
|
||||
type: string,
|
||||
payload: Record<string, unknown>,
|
||||
timeoutMs = 120_000,
|
||||
): Promise<AgentRequestResult> {
|
||||
const messageId = randomUUID();
|
||||
|
||||
await publishNodeCommand({
|
||||
nodeId,
|
||||
envelope: {
|
||||
protocolVersion: 1,
|
||||
messageId,
|
||||
type,
|
||||
timestamp: new Date().toISOString(),
|
||||
payload: {
|
||||
...payload,
|
||||
generation: payload['generation'] ?? 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const response = await waitForNodeResponse(messageId, timeoutMs);
|
||||
|
||||
if (response === null) {
|
||||
logger.error({ nodeId, type, messageId }, 'Agent request timed out');
|
||||
return {
|
||||
success: false,
|
||||
errorCode: 'AGENT_TIMEOUT',
|
||||
errorMessage: 'Agent request timed out',
|
||||
};
|
||||
}
|
||||
|
||||
return response as AgentRequestResult;
|
||||
}
|
||||
|
||||
export function loadStorageEnv(): void {
|
||||
validateConfig();
|
||||
}
|
||||
632
apps/worker/src/handlers/server-backups.ts
Normal file
632
apps/worker/src/handlers/server-backups.ts
Normal file
@@ -0,0 +1,632 @@
|
||||
import { Job } from 'bullmq';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { prisma } from '@hexahost/database';
|
||||
import {
|
||||
backupObjectKey,
|
||||
createPresignedGetUrl,
|
||||
createPresignedPutUrl,
|
||||
createS3Client,
|
||||
deleteStoredObject,
|
||||
loadStorageConfig,
|
||||
} from '@hexahost/storage';
|
||||
|
||||
import { sendAgentRequest } from '../agent-bridge';
|
||||
import { logger } from '../logger';
|
||||
import { transitionServerStatus } from '../server-state';
|
||||
import { getBackupsQueue } from '../queues';
|
||||
|
||||
const BACKUP_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
|
||||
const createBackupJobSchema = z.object({
|
||||
backupId: z.string().uuid(),
|
||||
serverId: z.string().uuid(),
|
||||
previousStatus: z.enum(['RUNNING', 'STOPPED']),
|
||||
});
|
||||
|
||||
const restoreBackupJobSchema = z.object({
|
||||
restoreId: z.string().uuid(),
|
||||
serverId: z.string().uuid(),
|
||||
backupId: z.string().uuid(),
|
||||
});
|
||||
|
||||
const applyWorldUploadJobSchema = z.object({
|
||||
uploadId: z.string().uuid(),
|
||||
serverId: z.string().uuid(),
|
||||
});
|
||||
|
||||
const retentionJobSchema = z.object({
|
||||
serverId: z.string().uuid(),
|
||||
});
|
||||
|
||||
interface ArchiveResult {
|
||||
localPath: string;
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
worldName: string;
|
||||
}
|
||||
|
||||
function getStorage() {
|
||||
const config = loadStorageConfig();
|
||||
return {
|
||||
config,
|
||||
client: createS3Client(config),
|
||||
};
|
||||
}
|
||||
|
||||
async function loadServer(serverId: string) {
|
||||
const server = await prisma.gameServer.findUnique({
|
||||
where: { id: serverId },
|
||||
include: { backupSchedule: true },
|
||||
});
|
||||
|
||||
if (!server?.nodeId) {
|
||||
throw new Error(`Server ${serverId} is not assigned to a node`);
|
||||
}
|
||||
|
||||
return server;
|
||||
}
|
||||
|
||||
async function markBackupFailed(
|
||||
backupId: string,
|
||||
reason: string,
|
||||
): Promise<void> {
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: backupId },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
failureReason: reason,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function runBackupArchive(
|
||||
serverId: string,
|
||||
nodeId: string,
|
||||
backupId: string,
|
||||
worldName: string,
|
||||
generation: number,
|
||||
): Promise<ArchiveResult> {
|
||||
const response = await sendAgentRequest(
|
||||
nodeId,
|
||||
'server.world.archive',
|
||||
{
|
||||
serverId,
|
||||
generation,
|
||||
worldName,
|
||||
archiveId: backupId,
|
||||
},
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (!response.success) {
|
||||
throw new Error(response.errorMessage ?? 'World archive failed');
|
||||
}
|
||||
|
||||
return response.result as ArchiveResult;
|
||||
}
|
||||
|
||||
async function uploadArchiveToS3(
|
||||
serverId: string,
|
||||
nodeId: string,
|
||||
backupId: string,
|
||||
archive: ArchiveResult,
|
||||
generation: number,
|
||||
): Promise<void> {
|
||||
const { config, client } = getStorage();
|
||||
const s3Key = backupObjectKey(serverId, backupId);
|
||||
const uploadUrl = await createPresignedPutUrl(
|
||||
client,
|
||||
config.S3_BUCKET,
|
||||
s3Key,
|
||||
60 * 60,
|
||||
);
|
||||
|
||||
const response = await sendAgentRequest(
|
||||
nodeId,
|
||||
'server.storage.upload',
|
||||
{
|
||||
serverId,
|
||||
generation,
|
||||
localPath: archive.localPath,
|
||||
uploadUrl,
|
||||
},
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (!response.success) {
|
||||
throw new Error(response.errorMessage ?? 'Backup upload failed');
|
||||
}
|
||||
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: backupId },
|
||||
data: {
|
||||
s3Key,
|
||||
sha256: archive.sha256,
|
||||
sizeBytes: BigInt(archive.sizeBytes),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function enforceRetention(serverId: string, retentionCount: number): Promise<void> {
|
||||
const backups = await prisma.serverBackup.findMany({
|
||||
where: {
|
||||
serverId,
|
||||
status: 'AVAILABLE',
|
||||
type: { in: ['MANUAL', 'SCHEDULED'] },
|
||||
},
|
||||
orderBy: { createdAt: 'desc' },
|
||||
});
|
||||
|
||||
const extras = backups.slice(retentionCount);
|
||||
if (extras.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { config, client } = getStorage();
|
||||
|
||||
for (const backup of extras) {
|
||||
if (backup.s3Key) {
|
||||
try {
|
||||
await deleteStoredObject(client, config.S3_BUCKET, backup.s3Key);
|
||||
} catch (error) {
|
||||
logger.warn({ backupId: backup.id, error }, 'Failed to delete S3 backup object');
|
||||
}
|
||||
}
|
||||
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: backup.id },
|
||||
data: { status: 'DELETED' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function processCreateBackupJob(job: Job): Promise<{ status: string }> {
|
||||
if (job.name !== 'create-backup') {
|
||||
return { status: 'ignored' };
|
||||
}
|
||||
|
||||
const { backupId, serverId, previousStatus } = createBackupJobSchema.parse(job.data);
|
||||
const server = await loadServer(serverId);
|
||||
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: backupId },
|
||||
data: { status: 'RUNNING' },
|
||||
});
|
||||
|
||||
try {
|
||||
if (previousStatus === 'RUNNING') {
|
||||
const prepare = await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.backup.prepare',
|
||||
{ serverId, generation: server.version },
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
if (!prepare.success) {
|
||||
throw new Error(prepare.errorMessage ?? 'Backup prepare failed');
|
||||
}
|
||||
}
|
||||
|
||||
const archive = await runBackupArchive(
|
||||
serverId,
|
||||
server.nodeId!,
|
||||
backupId,
|
||||
server.activeWorldName,
|
||||
server.version,
|
||||
);
|
||||
|
||||
await uploadArchiveToS3(
|
||||
serverId,
|
||||
server.nodeId!,
|
||||
backupId,
|
||||
archive,
|
||||
server.version,
|
||||
);
|
||||
|
||||
if (previousStatus === 'RUNNING') {
|
||||
const release = await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.backup.release',
|
||||
{ serverId, generation: server.version },
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
if (!release.success) {
|
||||
throw new Error(release.errorMessage ?? 'Backup release failed');
|
||||
}
|
||||
}
|
||||
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: backupId },
|
||||
data: {
|
||||
status: 'AVAILABLE',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await transitionServerStatus(serverId, previousStatus, {
|
||||
expectedFrom: 'BACKING_UP',
|
||||
reason: 'Backup completed',
|
||||
});
|
||||
|
||||
const retentionCount = server.backupSchedule?.retentionCount ?? 5;
|
||||
await enforceRetention(serverId, retentionCount);
|
||||
|
||||
return { status: 'available' };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Backup failed';
|
||||
await markBackupFailed(backupId, message);
|
||||
|
||||
try {
|
||||
if (previousStatus === 'RUNNING') {
|
||||
await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.backup.release',
|
||||
{ serverId, generation: server.version },
|
||||
60_000,
|
||||
);
|
||||
}
|
||||
await transitionServerStatus(serverId, previousStatus, {
|
||||
expectedFrom: 'BACKING_UP',
|
||||
reason: 'Backup failed',
|
||||
errorMessage: message,
|
||||
});
|
||||
} catch (restoreError) {
|
||||
logger.error({ restoreError, serverId }, 'Failed to restore server status after backup error');
|
||||
await transitionServerStatus(serverId, 'ERROR', {
|
||||
expectedFrom: 'BACKING_UP',
|
||||
reason: 'Backup failed and status recovery failed',
|
||||
errorMessage: message,
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function processRestoreBackupJob(job: Job): Promise<{ status: string }> {
|
||||
if (job.name !== 'restore-backup') {
|
||||
return { status: 'ignored' };
|
||||
}
|
||||
|
||||
const { restoreId, serverId, backupId } = restoreBackupJobSchema.parse(job.data);
|
||||
const server = await loadServer(serverId);
|
||||
const backup = await prisma.serverBackup.findUniqueOrThrow({
|
||||
where: { id: backupId },
|
||||
});
|
||||
|
||||
if (backup.status !== 'AVAILABLE' || !backup.s3Key) {
|
||||
throw new Error('Backup is not available for restore');
|
||||
}
|
||||
|
||||
await prisma.backupRestore.update({
|
||||
where: { id: restoreId },
|
||||
data: { status: 'RUNNING' },
|
||||
});
|
||||
|
||||
const safetyBackup = await prisma.serverBackup.create({
|
||||
data: {
|
||||
serverId,
|
||||
type: 'PRE_RESTORE',
|
||||
status: 'RUNNING',
|
||||
label: 'Pre-restore safety backup',
|
||||
},
|
||||
});
|
||||
|
||||
await prisma.backupRestore.update({
|
||||
where: { id: restoreId },
|
||||
data: { safetyBackupId: safetyBackup.id },
|
||||
});
|
||||
|
||||
try {
|
||||
const safetyArchive = await runBackupArchive(
|
||||
serverId,
|
||||
server.nodeId!,
|
||||
safetyBackup.id,
|
||||
server.activeWorldName,
|
||||
server.version,
|
||||
);
|
||||
await uploadArchiveToS3(
|
||||
serverId,
|
||||
server.nodeId!,
|
||||
safetyBackup.id,
|
||||
safetyArchive,
|
||||
server.version,
|
||||
);
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: safetyBackup.id },
|
||||
data: {
|
||||
status: 'AVAILABLE',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Safety backup failed';
|
||||
await markBackupFailed(safetyBackup.id, message);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { config, client } = getStorage();
|
||||
const downloadUrl = await createPresignedGetUrl(
|
||||
client,
|
||||
config.S3_BUCKET,
|
||||
backup.s3Key,
|
||||
60 * 60,
|
||||
);
|
||||
const localArchive = `/var/lib/hgc/servers/${serverId}/.hgc/restore/${restoreId}.tar.gz`;
|
||||
|
||||
try {
|
||||
const download = await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.storage.download',
|
||||
{
|
||||
serverId,
|
||||
generation: server.version,
|
||||
downloadUrl,
|
||||
localPath: localArchive,
|
||||
},
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (!download.success) {
|
||||
throw new Error(download.errorMessage ?? 'Backup download failed');
|
||||
}
|
||||
|
||||
const replace = await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.world.replace',
|
||||
{
|
||||
serverId,
|
||||
generation: server.version,
|
||||
worldName: server.activeWorldName,
|
||||
archivePath: localArchive,
|
||||
},
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (!replace.success) {
|
||||
throw new Error(replace.errorMessage ?? 'World replace failed');
|
||||
}
|
||||
|
||||
await prisma.backupRestore.update({
|
||||
where: { id: restoreId },
|
||||
data: {
|
||||
status: 'COMPLETED',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await transitionServerStatus(serverId, 'STOPPED', {
|
||||
expectedFrom: 'RESTORING',
|
||||
reason: 'Restore completed',
|
||||
});
|
||||
|
||||
return { status: 'completed' };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Restore failed';
|
||||
await prisma.backupRestore.update({
|
||||
where: { id: restoreId },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
failureReason: message,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
|
||||
await transitionServerStatus(serverId, 'STOPPED', {
|
||||
expectedFrom: 'RESTORING',
|
||||
reason: 'Restore failed',
|
||||
errorMessage: message,
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function processApplyWorldUploadJob(job: Job): Promise<{ status: string }> {
|
||||
if (job.name !== 'apply-world-upload') {
|
||||
return { status: 'ignored' };
|
||||
}
|
||||
|
||||
const { uploadId, serverId } = applyWorldUploadJobSchema.parse(job.data);
|
||||
const server = await loadServer(serverId);
|
||||
const upload = await prisma.worldUpload.findUniqueOrThrow({
|
||||
where: { id: uploadId },
|
||||
});
|
||||
|
||||
await prisma.worldUpload.update({
|
||||
where: { id: uploadId },
|
||||
data: { status: 'VALIDATING' },
|
||||
});
|
||||
|
||||
const safetyBackup = await prisma.serverBackup.create({
|
||||
data: {
|
||||
serverId,
|
||||
type: 'PRE_WORLD_REPLACE',
|
||||
status: 'RUNNING',
|
||||
label: `Pre-replace backup for upload ${uploadId}`,
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const safetyArchive = await runBackupArchive(
|
||||
serverId,
|
||||
server.nodeId!,
|
||||
safetyBackup.id,
|
||||
server.activeWorldName,
|
||||
server.version,
|
||||
);
|
||||
await uploadArchiveToS3(
|
||||
serverId,
|
||||
server.nodeId!,
|
||||
safetyBackup.id,
|
||||
safetyArchive,
|
||||
server.version,
|
||||
);
|
||||
await prisma.serverBackup.update({
|
||||
where: { id: safetyBackup.id },
|
||||
data: {
|
||||
status: 'AVAILABLE',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Safety backup failed';
|
||||
await markBackupFailed(safetyBackup.id, message);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { config, client } = getStorage();
|
||||
const downloadUrl = await createPresignedGetUrl(
|
||||
client,
|
||||
config.S3_BUCKET,
|
||||
upload.s3Key,
|
||||
60 * 60,
|
||||
);
|
||||
const localArchive = `/var/lib/hgc/servers/${serverId}/.hgc/uploads/${uploadId}.tar.gz`;
|
||||
|
||||
try {
|
||||
const download = await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.storage.download',
|
||||
{
|
||||
serverId,
|
||||
generation: server.version,
|
||||
downloadUrl,
|
||||
localPath: localArchive,
|
||||
},
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (!download.success) {
|
||||
throw new Error(download.errorMessage ?? 'Upload download failed');
|
||||
}
|
||||
|
||||
const replace = await sendAgentRequest(
|
||||
server.nodeId!,
|
||||
'server.world.replace',
|
||||
{
|
||||
serverId,
|
||||
generation: server.version,
|
||||
worldName: upload.worldName,
|
||||
archivePath: localArchive,
|
||||
},
|
||||
BACKUP_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
if (!replace.success) {
|
||||
throw new Error(replace.errorMessage ?? 'World replace failed');
|
||||
}
|
||||
|
||||
await prisma.$transaction([
|
||||
prisma.worldUpload.update({
|
||||
where: { id: uploadId },
|
||||
data: {
|
||||
status: 'COMPLETED',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
}),
|
||||
prisma.gameServer.update({
|
||||
where: { id: serverId },
|
||||
data: { activeWorldName: upload.worldName },
|
||||
}),
|
||||
]);
|
||||
|
||||
return { status: 'completed' };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'World upload apply failed';
|
||||
await prisma.worldUpload.update({
|
||||
where: { id: uploadId },
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
failureReason: message,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function processRetentionJob(job: Job): Promise<{ status: string }> {
|
||||
if (job.name !== 'enforce-retention') {
|
||||
return { status: 'ignored' };
|
||||
}
|
||||
|
||||
const { serverId } = retentionJobSchema.parse(job.data);
|
||||
const server = await prisma.gameServer.findUnique({
|
||||
where: { id: serverId },
|
||||
include: { backupSchedule: true },
|
||||
});
|
||||
|
||||
if (!server) {
|
||||
return { status: 'ignored' };
|
||||
}
|
||||
|
||||
await enforceRetention(serverId, server.backupSchedule?.retentionCount ?? 5);
|
||||
return { status: 'ok' };
|
||||
}
|
||||
|
||||
export async function processScheduledBackupsTick(): Promise<void> {
|
||||
const now = new Date();
|
||||
const schedules = await prisma.backupSchedule.findMany({
|
||||
where: {
|
||||
enabled: true,
|
||||
OR: [{ nextRunAt: null }, { nextRunAt: { lte: now } }],
|
||||
},
|
||||
});
|
||||
|
||||
for (const schedule of schedules) {
|
||||
const server = await prisma.gameServer.findUnique({
|
||||
where: { id: schedule.serverId },
|
||||
});
|
||||
|
||||
if (!server?.nodeId || !['RUNNING', 'STOPPED'].includes(server.status)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const backup = await prisma.serverBackup.create({
|
||||
data: {
|
||||
serverId: schedule.serverId,
|
||||
type: 'SCHEDULED',
|
||||
status: 'PENDING',
|
||||
label: 'Scheduled backup',
|
||||
},
|
||||
});
|
||||
|
||||
await transitionServerStatus(schedule.serverId, 'BACKING_UP', {
|
||||
expectedFrom: server.status as 'RUNNING' | 'STOPPED',
|
||||
reason: 'Scheduled backup started',
|
||||
});
|
||||
|
||||
await getBackupsQueue().add('create-backup', {
|
||||
backupId: backup.id,
|
||||
serverId: schedule.serverId,
|
||||
previousStatus: server.status,
|
||||
});
|
||||
|
||||
const nextRunAt = new Date(now.getTime() + schedule.intervalHours * 60 * 60 * 1000);
|
||||
await prisma.backupSchedule.update({
|
||||
where: { id: schedule.id },
|
||||
data: {
|
||||
lastRunAt: now,
|
||||
nextRunAt,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function processBackupJob(job: Job): Promise<{ status: string }> {
|
||||
switch (job.name) {
|
||||
case 'create-backup':
|
||||
return processCreateBackupJob(job);
|
||||
case 'restore-backup':
|
||||
return processRestoreBackupJob(job);
|
||||
case 'apply-world-upload':
|
||||
return processApplyWorldUploadJob(job);
|
||||
case 'enforce-retention':
|
||||
return processRetentionJob(job);
|
||||
default:
|
||||
logger.warn({ jobName: job.name }, 'Unknown backup job');
|
||||
return { status: 'ignored' };
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { validateConfig } from '@hexahost/config';
|
||||
import { prisma } from '@hexahost/database';
|
||||
|
||||
import { processNotificationJob } from './handlers/notifications';
|
||||
import { processBackupJob, processScheduledBackupsTick } from './handlers/server-backups';
|
||||
import { processLifecycleJob } from './handlers/server-lifecycle';
|
||||
import { processProvisionServerJob } from './handlers/server-provisioning';
|
||||
import { startHealthServer } from './health';
|
||||
@@ -11,6 +12,7 @@ import { logger } from './logger';
|
||||
import { closeRedisConnections } from './redis';
|
||||
import {
|
||||
QUEUE_NOTIFICATIONS,
|
||||
QUEUE_SERVER_BACKUPS,
|
||||
QUEUE_SERVER_LIFECYCLE,
|
||||
QUEUE_SERVER_PROVISIONING,
|
||||
WORKER_QUEUES,
|
||||
@@ -37,6 +39,10 @@ function createQueueWorker(
|
||||
return processProvisionServerJob(job);
|
||||
}
|
||||
|
||||
if (queueName === QUEUE_SERVER_BACKUPS) {
|
||||
return processBackupJob(job);
|
||||
}
|
||||
|
||||
if (queueName === QUEUE_SERVER_LIFECYCLE) {
|
||||
return processLifecycleJob(job);
|
||||
}
|
||||
@@ -75,9 +81,17 @@ async function bootstrap(): Promise<void> {
|
||||
|
||||
logger.info({ queues: WORKER_QUEUES }, 'Worker started');
|
||||
|
||||
const scheduleTimer = setInterval(() => {
|
||||
void processScheduledBackupsTick().catch((error: Error) => {
|
||||
logger.error({ err: error }, 'Scheduled backup tick failed');
|
||||
});
|
||||
}, 60_000);
|
||||
|
||||
const shutdown = async (signal: string): Promise<void> => {
|
||||
logger.info({ signal }, 'Shutting down worker');
|
||||
|
||||
clearInterval(scheduleTimer);
|
||||
|
||||
await Promise.all(workers.map((worker) => worker.close()));
|
||||
await closeRedisConnections();
|
||||
await prisma.$disconnect();
|
||||
|
||||
@@ -1,11 +1,33 @@
|
||||
export const QUEUE_SERVER_BACKUPS = 'server-backups' as const;
|
||||
export const QUEUE_SERVER_LIFECYCLE = 'server-lifecycle' as const;
|
||||
export const QUEUE_SERVER_PROVISIONING = 'server-provisioning' as const;
|
||||
export const QUEUE_NOTIFICATIONS = 'notifications' as const;
|
||||
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
import { validateConfig } from '@hexahost/config';
|
||||
|
||||
export const WORKER_QUEUES = [
|
||||
QUEUE_SERVER_BACKUPS,
|
||||
QUEUE_SERVER_LIFECYCLE,
|
||||
QUEUE_SERVER_PROVISIONING,
|
||||
QUEUE_NOTIFICATIONS,
|
||||
] as const;
|
||||
|
||||
export type WorkerQueueName = (typeof WORKER_QUEUES)[number];
|
||||
|
||||
let backupsQueue: Queue | null = null;
|
||||
|
||||
export function getBackupsQueue(): Queue {
|
||||
if (!backupsQueue) {
|
||||
const config = validateConfig();
|
||||
backupsQueue = new Queue(QUEUE_SERVER_BACKUPS, {
|
||||
connection: {
|
||||
url: config.REDIS_URL,
|
||||
maxRetriesPerRequest: null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return backupsQueue;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ export interface NodeCommandMessage {
|
||||
|
||||
export interface NodeResponseMessage {
|
||||
success: boolean;
|
||||
result?: unknown;
|
||||
resultCode?: string;
|
||||
errorCode?: string;
|
||||
errorMessage?: string;
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user