Phase8
This commit is contained in:
@@ -20,6 +20,7 @@
|
||||
"@hexahost/config": "workspace:*",
|
||||
"@hexahost/contracts": "workspace:*",
|
||||
"@hexahost/database": "workspace:*",
|
||||
"@hexahost/dns": "workspace:*",
|
||||
"@hexahost/metering": "workspace:*",
|
||||
"@hexahost/scheduler": "workspace:*",
|
||||
"@hexahost/storage": "workspace:*",
|
||||
|
||||
@@ -8,6 +8,7 @@ import { BillingModule } from './billing/billing.module';
|
||||
import { CatalogModule } from './catalog/catalog.module';
|
||||
import { AdminModule } from './admin/admin.module';
|
||||
import { NodesModule } from './nodes/nodes.module';
|
||||
import { EdgeModule } from './edge/edge.module';
|
||||
import { ServersModule } from './servers/servers.module';
|
||||
import { AppConfigModule } from './config/app-config.module';
|
||||
|
||||
@@ -44,6 +45,7 @@ import { LoggerModule } from 'nestjs-pino';
|
||||
AuthModule,
|
||||
CatalogModule,
|
||||
BillingModule,
|
||||
EdgeModule,
|
||||
ServersModule,
|
||||
NodesModule,
|
||||
AdminModule,
|
||||
|
||||
30
apps/api/src/edge/edge-internal.guard.ts
Normal file
30
apps/api/src/edge/edge-internal.guard.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import {
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
Injectable,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import type { FastifyRequest } from 'fastify';
|
||||
|
||||
const EDGE_KEY_HEADER = 'x-hexahost-edge-key';
|
||||
|
||||
function getEdgeInternalApiKey(): string {
|
||||
return (
|
||||
process.env['EDGE_INTERNAL_API_KEY'] ??
|
||||
'local-dev-edge-key-change-me-32chars'
|
||||
);
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class EdgeInternalGuard implements CanActivate {
|
||||
canActivate(context: ExecutionContext): boolean {
|
||||
const request = context.switchToHttp().getRequest<FastifyRequest>();
|
||||
const provided = request.headers[EDGE_KEY_HEADER];
|
||||
|
||||
if (typeof provided !== 'string' || provided !== getEdgeInternalApiKey()) {
|
||||
throw new UnauthorizedException('Invalid edge internal API key');
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
32
apps/api/src/edge/edge.controller.ts
Normal file
32
apps/api/src/edge/edge.controller.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import { Body, Controller, Get, Param, Post, Query, UseGuards } from '@nestjs/common';
|
||||
import { SkipThrottle } from '@nestjs/throttler';
|
||||
|
||||
import type { EdgeResolveResponse, EdgeStartResponse } from '@hexahost/contracts';
|
||||
import { edgeStartRequestSchema } from '@hexahost/contracts';
|
||||
|
||||
import { EdgeInternalGuard } from './edge-internal.guard';
|
||||
import { EdgeService } from './edge.service';
|
||||
|
||||
@Controller('internal/edge/v1')
|
||||
@UseGuards(EdgeInternalGuard)
|
||||
@SkipThrottle()
|
||||
export class EdgeController {
|
||||
constructor(private readonly edgeService: EdgeService) {}
|
||||
|
||||
@Get('resolve/:slug')
|
||||
resolve(
|
||||
@Param('slug') slug: string,
|
||||
@Query('clientIp') clientIp?: string,
|
||||
): Promise<EdgeResolveResponse> {
|
||||
return this.edgeService.resolveSlug(slug, clientIp);
|
||||
}
|
||||
|
||||
@Post('start/:slug')
|
||||
start(
|
||||
@Param('slug') slug: string,
|
||||
@Body() body: unknown,
|
||||
): Promise<EdgeStartResponse> {
|
||||
const parsed = edgeStartRequestSchema.parse(body);
|
||||
return this.edgeService.requestJoinStart(slug, parsed.clientIp);
|
||||
}
|
||||
}
|
||||
16
apps/api/src/edge/edge.module.ts
Normal file
16
apps/api/src/edge/edge.module.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { ServersModule } from '../servers/servers.module';
|
||||
|
||||
import { EdgeController } from './edge.controller';
|
||||
import { EdgeInternalGuard } from './edge-internal.guard';
|
||||
import { EdgeService } from './edge.service';
|
||||
import { JoinService } from './join.service';
|
||||
|
||||
@Module({
|
||||
imports: [ServersModule],
|
||||
controllers: [EdgeController],
|
||||
providers: [EdgeService, JoinService, EdgeInternalGuard],
|
||||
exports: [JoinService, EdgeService],
|
||||
})
|
||||
export class EdgeModule {}
|
||||
199
apps/api/src/edge/edge.service.ts
Normal file
199
apps/api/src/edge/edge.service.ts
Normal file
@@ -0,0 +1,199 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Inject,
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
} from '@nestjs/common';
|
||||
import type Redis from 'ioredis';
|
||||
|
||||
import type { EdgeResolveResponse, EdgeStartResponse } from '@hexahost/contracts';
|
||||
import type { GameServer } from '@hexahost/database';
|
||||
import { getPlayDomain } from '@hexahost/dns';
|
||||
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { ServersService, REDIS_CLIENT } from '../servers/servers.service';
|
||||
|
||||
import { JoinService } from './join.service';
|
||||
|
||||
const STARTABLE_STATUSES = new Set<GameServer['status']>(['STOPPED', 'UNKNOWN']);
|
||||
const WAIT_STATUSES = new Set<GameServer['status']>([
|
||||
'STARTING',
|
||||
'QUEUED',
|
||||
'PROVISIONING',
|
||||
'INSTALLING',
|
||||
'STOPPING',
|
||||
]);
|
||||
const REJECT_STATUSES = new Set<GameServer['status']>([
|
||||
'DRAFT',
|
||||
'ERROR',
|
||||
'DELETING',
|
||||
'DELETED',
|
||||
'BACKING_UP',
|
||||
'RESTORING',
|
||||
]);
|
||||
|
||||
@Injectable()
|
||||
export class EdgeService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly joinService: JoinService,
|
||||
private readonly serversService: ServersService,
|
||||
@Inject(REDIS_CLIENT) private readonly redis: Redis,
|
||||
) {}
|
||||
|
||||
async resolveSlug(slug: string, clientIp?: string): Promise<EdgeResolveResponse> {
|
||||
const server = await this.prisma.gameServer.findFirst({
|
||||
where: { joinSlug: slug },
|
||||
include: { node: true },
|
||||
});
|
||||
|
||||
if (!server) {
|
||||
throw new NotFoundException(`Unknown join slug: ${slug}`);
|
||||
}
|
||||
|
||||
return this.buildResolveResponse(server, clientIp);
|
||||
}
|
||||
|
||||
async requestJoinStart(
|
||||
slug: string,
|
||||
clientIp: string,
|
||||
): Promise<EdgeStartResponse> {
|
||||
if (!(await this.checkStartRateLimit(slug, clientIp))) {
|
||||
throw new HttpException(
|
||||
'Join-to-start rate limit exceeded',
|
||||
HttpStatus.TOO_MANY_REQUESTS,
|
||||
);
|
||||
}
|
||||
|
||||
const server = await this.prisma.gameServer.findFirst({
|
||||
where: { joinSlug: slug },
|
||||
include: { node: true },
|
||||
});
|
||||
|
||||
if (!server) {
|
||||
throw new NotFoundException(`Unknown join slug: ${slug}`);
|
||||
}
|
||||
|
||||
if (!server.joinToStartEnabled) {
|
||||
throw new BadRequestException('Join-to-start is disabled for this server');
|
||||
}
|
||||
|
||||
const initial = await this.buildResolveResponse(server, clientIp);
|
||||
|
||||
if (initial.action === 'proxy') {
|
||||
return { resolve: initial, started: false };
|
||||
}
|
||||
|
||||
if (initial.action === 'reject') {
|
||||
return { resolve: initial, started: false };
|
||||
}
|
||||
|
||||
if (initial.action === 'wait') {
|
||||
return { resolve: initial, started: false };
|
||||
}
|
||||
|
||||
const started = await this.serversService.startServerInternal(
|
||||
server.id,
|
||||
`join-to-start:${clientIp}`,
|
||||
);
|
||||
|
||||
const refreshed = await this.prisma.gameServer.findUniqueOrThrow({
|
||||
where: { id: server.id },
|
||||
include: { node: true },
|
||||
});
|
||||
|
||||
return {
|
||||
resolve: await this.buildResolveResponse(refreshed, clientIp),
|
||||
started: true,
|
||||
jobId: started.jobId,
|
||||
queuePosition: started.queuePosition,
|
||||
};
|
||||
}
|
||||
|
||||
private async buildResolveResponse(
|
||||
server: GameServer & { node: { hostname: string } | null },
|
||||
clientIp?: string,
|
||||
): Promise<EdgeResolveResponse> {
|
||||
const backend =
|
||||
server.node && server.hostPort !== null
|
||||
? { host: server.node.hostname, port: server.hostPort }
|
||||
: null;
|
||||
|
||||
const base = {
|
||||
serverId: server.id,
|
||||
slug: server.joinSlug ?? '',
|
||||
status: server.status,
|
||||
joinToStartEnabled: server.joinToStartEnabled,
|
||||
edition: server.edition,
|
||||
backend,
|
||||
};
|
||||
|
||||
if (REJECT_STATUSES.has(server.status)) {
|
||||
return {
|
||||
...base,
|
||||
action: 'reject',
|
||||
message: `Server is ${server.status.toLowerCase()}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (server.status === 'RUNNING' && backend) {
|
||||
return { ...base, action: 'proxy', backend };
|
||||
}
|
||||
|
||||
if (!server.joinToStartEnabled) {
|
||||
return {
|
||||
...base,
|
||||
action: 'reject',
|
||||
message: 'Join-to-start is disabled; start the server from the panel',
|
||||
};
|
||||
}
|
||||
|
||||
if (WAIT_STATUSES.has(server.status)) {
|
||||
return {
|
||||
...base,
|
||||
action: 'wait',
|
||||
message: `Server is ${server.status.toLowerCase()}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (STARTABLE_STATUSES.has(server.status)) {
|
||||
return {
|
||||
...base,
|
||||
action: 'start',
|
||||
message: clientIp ? `Start requested from ${clientIp}` : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...base,
|
||||
action: 'reject',
|
||||
message: `Cannot join while server is ${server.status.toLowerCase()}`,
|
||||
};
|
||||
}
|
||||
|
||||
private async checkStartRateLimit(slug: string, clientIp: string): Promise<boolean> {
|
||||
const slugKey = `edge:start:slug:${slug}`;
|
||||
const ipKey = `edge:start:ip:${clientIp}`;
|
||||
|
||||
const slugCount = await this.redis.incr(slugKey);
|
||||
if (slugCount === 1) {
|
||||
await this.redis.expire(slugKey, 60);
|
||||
}
|
||||
if (slugCount > 3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ipCount = await this.redis.incr(ipKey);
|
||||
if (ipCount === 1) {
|
||||
await this.redis.expire(ipKey, 60);
|
||||
}
|
||||
|
||||
return ipCount <= 10;
|
||||
}
|
||||
|
||||
getPlayDomain(): string {
|
||||
return getPlayDomain();
|
||||
}
|
||||
}
|
||||
31
apps/api/src/edge/join.service.ts
Normal file
31
apps/api/src/edge/join.service.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
|
||||
import { assignJoinAddress, formatJoinAddress, getPlayDomain } from '@hexahost/dns';
|
||||
import type { GameServer, ServerDnsRecord } from '@hexahost/database';
|
||||
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
|
||||
@Injectable()
|
||||
export class JoinService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
getPlayDomain(): string {
|
||||
return getPlayDomain();
|
||||
}
|
||||
|
||||
formatJoinHostname(server: Pick<GameServer, 'joinSlug' | 'edition'>): string | null {
|
||||
return formatJoinAddress(server.joinSlug, server.edition);
|
||||
}
|
||||
|
||||
async ensureJoinAddress(serverId: string): Promise<string | null> {
|
||||
const result = await assignJoinAddress(serverId);
|
||||
return result?.joinHostname ?? null;
|
||||
}
|
||||
|
||||
async getDnsRecords(serverId: string): Promise<ServerDnsRecord[]> {
|
||||
return this.prisma.serverDnsRecord.findMany({
|
||||
where: { serverId },
|
||||
orderBy: { recordType: 'asc' },
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import type {
|
||||
ServerResponse,
|
||||
} from '@hexahost/contracts';
|
||||
import type { GameServer } from '@hexahost/database';
|
||||
import { formatJoinAddress } from '@hexahost/dns';
|
||||
|
||||
type GameServerStatus = GameServer['status'];
|
||||
|
||||
@@ -177,6 +178,89 @@ export class ServersService {
|
||||
};
|
||||
}
|
||||
|
||||
async startServerInternal(
|
||||
serverId: string,
|
||||
reason: string,
|
||||
): Promise<ServerActionResponse> {
|
||||
const server = await this.prisma.gameServer.findUnique({
|
||||
where: { id: serverId },
|
||||
});
|
||||
|
||||
if (!server) {
|
||||
throw new ConflictException('Server not found');
|
||||
}
|
||||
|
||||
this.serverState.assertNotInProgress(server.status);
|
||||
|
||||
const targetStatus = this.serverState.resolveStartTarget(server.status);
|
||||
this.serverState.assertTransition(server.status, targetStatus);
|
||||
|
||||
const correlationId = randomUUID();
|
||||
|
||||
if (targetStatus === 'STARTING') {
|
||||
await this.billing.assertCanStartServer(server.userId);
|
||||
|
||||
const canStart = await this.scheduling.canStartServer(server);
|
||||
if (!canStart) {
|
||||
const queued = await this.scheduling.enqueueStart(
|
||||
server,
|
||||
server.userId,
|
||||
correlationId,
|
||||
);
|
||||
return {
|
||||
server: this.toResponse(queued.server),
|
||||
queuePosition: queued.queuePosition,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const updated = await this.transitionServer(
|
||||
server,
|
||||
targetStatus,
|
||||
reason,
|
||||
null,
|
||||
correlationId,
|
||||
);
|
||||
|
||||
let jobId: string | undefined;
|
||||
|
||||
if (targetStatus === 'PROVISIONING') {
|
||||
const job = await this.provisioningQueue.add(
|
||||
'provision-server',
|
||||
{ serverId, correlationId },
|
||||
{ jobId: correlationId },
|
||||
);
|
||||
jobId = job.id;
|
||||
} else {
|
||||
const job = await this.lifecycleQueue.add(
|
||||
'start-server',
|
||||
{ serverId, action: 'start', correlationId },
|
||||
{ jobId: correlationId },
|
||||
);
|
||||
jobId = job.id;
|
||||
|
||||
if (updated.nodeId) {
|
||||
await this.publishNodeCommand(updated.nodeId, {
|
||||
protocolVersion: 1,
|
||||
messageId: randomUUID(),
|
||||
correlationId,
|
||||
type: 'server.start',
|
||||
timestamp: new Date().toISOString(),
|
||||
payload: {
|
||||
serverId,
|
||||
generation: updated.version,
|
||||
desiredGeneration: updated.version,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
server: this.toResponse(updated),
|
||||
jobId,
|
||||
};
|
||||
}
|
||||
|
||||
async getStartQueue(userId: string, serverId: string) {
|
||||
return this.scheduling.getStartQueue(userId, serverId);
|
||||
}
|
||||
@@ -370,6 +454,9 @@ export class ServersService {
|
||||
ramMb: server.ramMb,
|
||||
idleShutdownEnabled: server.idleShutdownEnabled,
|
||||
idleStopAt: server.idleStopAt?.toISOString() ?? null,
|
||||
joinSlug: server.joinSlug,
|
||||
joinHostname: formatJoinAddress(server.joinSlug, server.edition),
|
||||
joinToStartEnabled: server.joinToStartEnabled,
|
||||
createdAt: server.createdAt.toISOString(),
|
||||
updatedAt: server.updatedAt.toISOString(),
|
||||
};
|
||||
|
||||
30
apps/api/test/phase8.test.js
Normal file
30
apps/api/test/phase8.test.js
Normal file
@@ -0,0 +1,30 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
describe('phase 8 api contract', () => {
|
||||
it('documents internal edge routes and join fields', () => {
|
||||
const edgeSource = readFileSync(
|
||||
join(process.cwd(), 'src', 'edge/edge.controller.ts'),
|
||||
'utf8',
|
||||
);
|
||||
assert.ok(edgeSource.includes("'resolve/:slug'"));
|
||||
assert.ok(edgeSource.includes("'start/:slug'"));
|
||||
assert.ok(edgeSource.includes('EdgeInternalGuard'));
|
||||
|
||||
const edgeServiceSource = readFileSync(
|
||||
join(process.cwd(), 'src', 'edge/edge.service.ts'),
|
||||
'utf8',
|
||||
);
|
||||
assert.ok(edgeServiceSource.includes('requestJoinStart'));
|
||||
assert.ok(edgeServiceSource.includes('checkStartRateLimit'));
|
||||
|
||||
const serversSource = readFileSync(
|
||||
join(process.cwd(), 'src', 'servers/servers.service.ts'),
|
||||
'utf8',
|
||||
);
|
||||
assert.ok(serversSource.includes('startServerInternal'));
|
||||
assert.ok(serversSource.includes('joinHostname'));
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/hexahost/gamecloud/edge-gateway/internal/config"
|
||||
"github.com/hexahost/gamecloud/edge-gateway/internal/proxy"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -12,18 +18,21 @@ func main() {
|
||||
|
||||
enabled := os.Getenv("EDGE_GATEWAY_ENABLED")
|
||||
if enabled != "true" && enabled != "1" {
|
||||
log.Info("edge gateway disabled for MVP",
|
||||
log.Info("edge gateway disabled",
|
||||
"feature", "edge-gateway",
|
||||
"status", "not_implemented",
|
||||
"message", "Join-to-Start TCP/UDP gateway is planned for Phase 8; set EDGE_GATEWAY_ENABLED=true to acknowledge early enablement",
|
||||
"hint", "set EDGE_GATEWAY_ENABLED=true to start the TCP join-to-start gateway",
|
||||
)
|
||||
os.Exit(0)
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("edge gateway not implemented in MVP",
|
||||
"feature", "edge-gateway",
|
||||
"status", "not_implemented",
|
||||
"roadmap", "Phase 8 – DNS and Join-to-Start",
|
||||
)
|
||||
os.Exit(0)
|
||||
cfg := config.Load()
|
||||
gateway := proxy.New(cfg, log)
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if err := gateway.ListenAndServe(ctx); err != nil {
|
||||
log.Error("edge gateway stopped", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
37
apps/edge-gateway/internal/config/config.go
Normal file
37
apps/edge-gateway/internal/config/config.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
ListenAddr string
|
||||
APIURL string
|
||||
EdgeKey string
|
||||
PlayDomain string
|
||||
StartWait time.Duration
|
||||
PollInterval time.Duration
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
startWaitSec, _ := strconv.Atoi(envOr("EDGE_START_WAIT_SECONDS", "180"))
|
||||
pollMs, _ := strconv.Atoi(envOr("EDGE_POLL_INTERVAL_MS", "2000"))
|
||||
|
||||
return Config{
|
||||
ListenAddr: envOr("EDGE_LISTEN_ADDR", ":25565"),
|
||||
APIURL: envOr("API_URL", "http://localhost:3001"),
|
||||
EdgeKey: envOr("EDGE_INTERNAL_API_KEY", "local-dev-edge-key-change-me-32chars"),
|
||||
PlayDomain: envOr("GAME_BASE_DOMAIN", "play.example.net"),
|
||||
StartWait: time.Duration(startWaitSec) * time.Second,
|
||||
PollInterval: time.Duration(pollMs) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
func envOr(key, fallback string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
119
apps/edge-gateway/internal/controlplane/client.go
Normal file
119
apps/edge-gateway/internal/controlplane/client.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Backend struct {
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
}
|
||||
|
||||
type ResolveResponse struct {
|
||||
ServerID string `json:"serverId"`
|
||||
Slug string `json:"slug"`
|
||||
Status string `json:"status"`
|
||||
JoinToStartEnabled bool `json:"joinToStartEnabled"`
|
||||
Edition string `json:"edition"`
|
||||
Backend *Backend `json:"backend"`
|
||||
Action string `json:"action"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
type StartResponse struct {
|
||||
Resolve ResolveResponse `json:"resolve"`
|
||||
Started bool `json:"started"`
|
||||
JobID string `json:"jobId,omitempty"`
|
||||
QueuePosition int `json:"queuePosition,omitempty"`
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
edgeKey string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func NewClient(baseURL, edgeKey string) *Client {
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
edgeKey: edgeKey,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) Resolve(slug, clientIP string) (*ResolveResponse, error) {
|
||||
endpoint := fmt.Sprintf("%s/internal/edge/v1/resolve/%s", c.baseURL, url.PathEscape(slug))
|
||||
if clientIP != "" {
|
||||
endpoint = endpoint + "?clientIp=" + url.QueryEscape(clientIP)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.applyHeaders(req)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return nil, fmt.Errorf("unknown slug")
|
||||
}
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("resolve failed: %s", string(body))
|
||||
}
|
||||
|
||||
var result ResolveResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) Start(slug, clientIP string) (*StartResponse, error) {
|
||||
endpoint := fmt.Sprintf("%s/internal/edge/v1/start/%s", c.baseURL, url.PathEscape(slug))
|
||||
payload, err := json.Marshal(map[string]string{"clientIp": clientIP})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.applyHeaders(req)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("start failed: %s", string(body))
|
||||
}
|
||||
|
||||
var result StartResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) applyHeaders(req *http.Request) {
|
||||
req.Header.Set("X-HexaHost-Edge-Key", c.edgeKey)
|
||||
}
|
||||
174
apps/edge-gateway/internal/mc/handshake.go
Normal file
174
apps/edge-gateway/internal/mc/handshake.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package mc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Handshake struct {
|
||||
ProtocolVersion int32
|
||||
ServerAddress string
|
||||
ServerPort uint16
|
||||
NextState int32
|
||||
Raw []byte
|
||||
}
|
||||
|
||||
func ReadHandshake(r io.Reader) (*Handshake, error) {
|
||||
raw, err := readPacket(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader := newReader(raw)
|
||||
packetID, err := reader.readVarInt()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if packetID != 0 {
|
||||
return nil, fmt.Errorf("unexpected packet id %d", packetID)
|
||||
}
|
||||
|
||||
protocolVersion, err := reader.readVarInt()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serverAddress, err := reader.readString()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
portBytes, err := reader.readBytes(2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
serverPort := binary.BigEndian.Uint16(portBytes)
|
||||
|
||||
nextState, err := reader.readVarInt()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Handshake{
|
||||
ProtocolVersion: protocolVersion,
|
||||
ServerAddress: serverAddress,
|
||||
ServerPort: serverPort,
|
||||
NextState: nextState,
|
||||
Raw: raw,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ExtractSlug(hostname, playDomain string) string {
|
||||
host := strings.ToLower(strings.Split(hostname, "\x00")[0])
|
||||
domain := strings.ToLower(playDomain)
|
||||
suffix := "." + domain
|
||||
if !strings.HasSuffix(host, suffix) {
|
||||
return ""
|
||||
}
|
||||
slug := strings.TrimSuffix(host, suffix)
|
||||
if slug == "" || strings.Contains(slug, ".") {
|
||||
return ""
|
||||
}
|
||||
return slug
|
||||
}
|
||||
|
||||
func readPacket(r io.Reader) ([]byte, error) {
|
||||
length, err := readVarIntFrom(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length <= 0 || length > 1<<20 {
|
||||
return nil, fmt.Errorf("invalid packet length %d", length)
|
||||
}
|
||||
|
||||
payload := make([]byte, length)
|
||||
if _, err := io.ReadFull(r, payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
type packetReader struct {
|
||||
buf *bytes.Reader
|
||||
}
|
||||
|
||||
func newReader(data []byte) *packetReader {
|
||||
return &packetReader{buf: bytes.NewReader(data)}
|
||||
}
|
||||
|
||||
func (p *packetReader) readVarInt() (int32, error) {
|
||||
return readVarIntFrom(p.buf)
|
||||
}
|
||||
|
||||
func (p *packetReader) readString() (string, error) {
|
||||
length, err := p.readVarInt()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if length < 0 || length > 255 {
|
||||
return "", fmt.Errorf("invalid string length %d", length)
|
||||
}
|
||||
data, err := p.readBytes(int(length))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func (p *packetReader) readBytes(n int) ([]byte, error) {
|
||||
out := make([]byte, n)
|
||||
if _, err := io.ReadFull(p.buf, out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func readVarIntFrom(r io.Reader) (int32, error) {
|
||||
var numRead int
|
||||
var result int32
|
||||
|
||||
for {
|
||||
var value [1]byte
|
||||
if _, err := io.ReadFull(r, value[:]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
result |= int32(value[0]&0x7F) << (7 * numRead)
|
||||
numRead++
|
||||
if numRead > 5 {
|
||||
return 0, fmt.Errorf("varint too big")
|
||||
}
|
||||
if value[0]&0x80 == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func WritePacket(w io.Writer, payload []byte) error {
|
||||
if err := writeVarIntTo(w, int32(len(payload))); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.Write(payload)
|
||||
return err
|
||||
}
|
||||
|
||||
func writeVarIntTo(w io.Writer, value int32) error {
|
||||
for {
|
||||
temp := byte(value & 0x7F)
|
||||
value >>= 7
|
||||
if value != 0 {
|
||||
temp |= 0x80
|
||||
}
|
||||
if _, err := w.Write([]byte{temp}); err != nil {
|
||||
return err
|
||||
}
|
||||
if value == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
59
apps/edge-gateway/internal/mc/handshake_test.go
Normal file
59
apps/edge-gateway/internal/mc/handshake_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package mc
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractSlug(t *testing.T) {
|
||||
if got := ExtractSlug("abc.play.example.net", "play.example.net"); got != "abc" {
|
||||
t.Fatalf("got %q want abc", got)
|
||||
}
|
||||
if got := ExtractSlug("other.net", "play.example.net"); got != "" {
|
||||
t.Fatalf("expected empty slug")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHandshake(t *testing.T) {
|
||||
var payload bytes.Buffer
|
||||
writeVarInt(&payload, 0)
|
||||
writeVarInt(&payload, 763)
|
||||
writeString(&payload, "demo.play.example.net")
|
||||
payload.WriteByte(0x63)
|
||||
payload.WriteByte(0xdd)
|
||||
writeVarInt(&payload, 2)
|
||||
|
||||
var packet bytes.Buffer
|
||||
writeVarInt(&packet, int32(payload.Len()))
|
||||
packet.Write(payload.Bytes())
|
||||
|
||||
hs, err := ReadHandshake(&packet)
|
||||
if err != nil {
|
||||
t.Fatalf("read handshake: %v", err)
|
||||
}
|
||||
if hs.ServerAddress != "demo.play.example.net" {
|
||||
t.Fatalf("address %q", hs.ServerAddress)
|
||||
}
|
||||
if hs.ServerPort != 25565 {
|
||||
t.Fatalf("port %d", hs.ServerPort)
|
||||
}
|
||||
}
|
||||
|
||||
func writeVarInt(buf *bytes.Buffer, value int32) {
|
||||
for {
|
||||
temp := byte(value & 0x7F)
|
||||
value >>= 7
|
||||
if value != 0 {
|
||||
temp |= 0x80
|
||||
}
|
||||
buf.WriteByte(temp)
|
||||
if value == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeString(buf *bytes.Buffer, value string) {
|
||||
writeVarInt(buf, int32(len(value)))
|
||||
buf.WriteString(value)
|
||||
}
|
||||
160
apps/edge-gateway/internal/proxy/gateway.go
Normal file
160
apps/edge-gateway/internal/proxy/gateway.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/hexahost/gamecloud/edge-gateway/internal/config"
|
||||
"github.com/hexahost/gamecloud/edge-gateway/internal/controlplane"
|
||||
"github.com/hexahost/gamecloud/edge-gateway/internal/mc"
|
||||
)
|
||||
|
||||
type Gateway struct {
|
||||
cfg config.Config
|
||||
client *controlplane.Client
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func New(cfg config.Config, log *slog.Logger) *Gateway {
|
||||
return &Gateway{
|
||||
cfg: cfg,
|
||||
client: controlplane.NewClient(cfg.APIURL, cfg.EdgeKey),
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gateway) ListenAndServe(ctx context.Context) error {
|
||||
listener, err := net.Listen("tcp", g.cfg.ListenAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
g.log.Info("edge gateway listening",
|
||||
"addr", g.cfg.ListenAddr,
|
||||
"playDomain", g.cfg.PlayDomain,
|
||||
"api", g.cfg.APIURL,
|
||||
)
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
_ = listener.Close()
|
||||
}()
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
g.log.Error("accept failed", "error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
go g.handleConnection(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gateway) handleConnection(clientConn net.Conn) {
|
||||
defer clientConn.Close()
|
||||
|
||||
clientIP := clientConn.RemoteAddr().String()
|
||||
if host, _, err := net.SplitHostPort(clientIP); err == nil {
|
||||
clientIP = host
|
||||
}
|
||||
|
||||
handshake, err := mc.ReadHandshake(clientConn)
|
||||
if err != nil {
|
||||
g.log.Warn("handshake failed", "error", err, "client", clientIP)
|
||||
return
|
||||
}
|
||||
|
||||
slug := mc.ExtractSlug(handshake.ServerAddress, g.cfg.PlayDomain)
|
||||
if slug == "" {
|
||||
g.log.Warn("unknown hostname", "address", handshake.ServerAddress, "client", clientIP)
|
||||
return
|
||||
}
|
||||
|
||||
resolve, err := g.client.Resolve(slug, clientIP)
|
||||
if err != nil {
|
||||
g.log.Warn("resolve failed", "slug", slug, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
switch resolve.Action {
|
||||
case "start":
|
||||
startResp, err := g.client.Start(slug, clientIP)
|
||||
if err != nil {
|
||||
g.log.Warn("join start failed", "slug", slug, "error", err)
|
||||
return
|
||||
}
|
||||
resolve = &startResp.Resolve
|
||||
case "wait":
|
||||
resolve, err = g.waitForProxy(slug, clientIP)
|
||||
if err != nil {
|
||||
g.log.Warn("wait for server failed", "slug", slug, "error", err)
|
||||
return
|
||||
}
|
||||
case "reject":
|
||||
g.log.Info("connection rejected", "slug", slug, "message", resolve.Message)
|
||||
return
|
||||
}
|
||||
|
||||
if resolve.Action != "proxy" || resolve.Backend == nil {
|
||||
g.log.Warn("no backend available", "slug", slug, "action", resolve.Action)
|
||||
return
|
||||
}
|
||||
|
||||
backendAddr := fmt.Sprintf("%s:%d", resolve.Backend.Host, resolve.Backend.Port)
|
||||
backendConn, err := net.DialTimeout("tcp", backendAddr, 10*time.Second)
|
||||
if err != nil {
|
||||
g.log.Error("backend dial failed", "slug", slug, "backend", backendAddr, "error", err)
|
||||
return
|
||||
}
|
||||
defer backendConn.Close()
|
||||
|
||||
if err := mc.WritePacket(backendConn, handshake.Raw); err != nil {
|
||||
g.log.Error("replay handshake failed", "slug", slug, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
g.log.Info("proxying connection",
|
||||
"slug", slug,
|
||||
"client", clientIP,
|
||||
"backend", backendAddr,
|
||||
"status", resolve.Status,
|
||||
)
|
||||
|
||||
pipe(clientConn, backendConn)
|
||||
}
|
||||
|
||||
func (g *Gateway) waitForProxy(slug, clientIP string) (*controlplane.ResolveResponse, error) {
|
||||
deadline := time.Now().Add(g.cfg.StartWait)
|
||||
for time.Now().Before(deadline) {
|
||||
resolve, err := g.client.Resolve(slug, clientIP)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolve.Action == "proxy" {
|
||||
return resolve, nil
|
||||
}
|
||||
if resolve.Action == "reject" {
|
||||
return resolve, fmt.Errorf(resolve.Message)
|
||||
}
|
||||
time.Sleep(g.cfg.PollInterval)
|
||||
}
|
||||
return nil, fmt.Errorf("timed out waiting for server to start")
|
||||
}
|
||||
|
||||
func pipe(left, right net.Conn) {
|
||||
errCh := make(chan error, 2)
|
||||
go func() { _, err := io.Copy(right, left); errCh <- err }()
|
||||
go func() { _, err := io.Copy(left, right); errCh <- err }()
|
||||
<-errCh
|
||||
}
|
||||
@@ -61,7 +61,8 @@ export function ServerCard({ server, showDetailsLink = true }: ServerCardProps)
|
||||
server.status,
|
||||
);
|
||||
const address =
|
||||
server.hostPort !== null ? `localhost:${server.hostPort}` : t("addressPending");
|
||||
server.joinHostname ??
|
||||
(server.hostPort !== null ? `localhost:${server.hostPort}` : t("addressPending"));
|
||||
|
||||
return (
|
||||
<Card>
|
||||
|
||||
@@ -42,6 +42,9 @@ export interface Server {
|
||||
ramMb: number;
|
||||
idleShutdownEnabled: boolean;
|
||||
idleStopAt: string | null;
|
||||
joinSlug: string | null;
|
||||
joinHostname: string | null;
|
||||
joinToStartEnabled: boolean;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
"@hexahost/catalog": "workspace:*",
|
||||
"@hexahost/config": "workspace:*",
|
||||
"@hexahost/database": "workspace:*",
|
||||
"@hexahost/dns": "workspace:*",
|
||||
"@hexahost/metering": "workspace:*",
|
||||
"@hexahost/scheduler": "workspace:*",
|
||||
"@hexahost/storage": "workspace:*",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
import { prisma } from '@hexahost/database';
|
||||
import { assignJoinAddress } from '@hexahost/dns';
|
||||
import { selectNodeForAllocation } from '@hexahost/scheduler';
|
||||
import { Job } from 'bullmq';
|
||||
import { z } from 'zod';
|
||||
@@ -11,7 +12,6 @@ import { loadNodeSnapshots } from '../node-snapshots';
|
||||
import { publishNodeCommand, waitForNodeResponse } from '../redis';
|
||||
import { transitionServerStatus } from '../server-state';
|
||||
|
||||
const BASE_HOST_PORT = 25565;
|
||||
const PROVISION_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
|
||||
export const provisionServerJobSchema = z.object({
|
||||
@@ -23,6 +23,11 @@ function buildDataPath(serverId: string): string {
|
||||
}
|
||||
|
||||
async function findAvailableHostPort(nodeId: string): Promise<number> {
|
||||
const node = await prisma.gameNode.findUniqueOrThrow({
|
||||
where: { id: nodeId },
|
||||
select: { portRangeStart: true, portRangeEnd: true },
|
||||
});
|
||||
|
||||
const [allocations, servers] = await Promise.all([
|
||||
prisma.gameServerAllocation.findMany({
|
||||
where: { nodeId, status: 'ACTIVE' },
|
||||
@@ -44,9 +49,12 @@ async function findAvailableHostPort(nodeId: string): Promise<number> {
|
||||
}
|
||||
}
|
||||
|
||||
let port = BASE_HOST_PORT;
|
||||
let port = node.portRangeStart;
|
||||
while (usedPorts.has(port)) {
|
||||
port += 1;
|
||||
if (port > node.portRangeEnd) {
|
||||
throw new Error(`No free host port in range ${node.portRangeStart}-${node.portRangeEnd}`);
|
||||
}
|
||||
}
|
||||
|
||||
return port;
|
||||
@@ -171,6 +179,8 @@ export async function processProvisionServerJob(job: Job): Promise<{ status: str
|
||||
correlationId: messageId,
|
||||
});
|
||||
|
||||
await assignJoinAddress(serverId);
|
||||
|
||||
logger.info({ serverId, nodeId: node.id, hostPort }, 'Server provisioned');
|
||||
return { status: 'stopped' };
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user