Phase2
This commit is contained in:
@@ -22,7 +22,13 @@ func main() {
|
||||
|
||||
log = log.With("node_id", cfg.NodeID, "component", "node-agent")
|
||||
|
||||
if err := agent.New(cfg, log).Run(context.Background()); err != nil {
|
||||
a, err := agent.New(cfg, log)
|
||||
if err != nil {
|
||||
log.Error("agent init error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := a.Run(context.Background()); err != nil {
|
||||
log.Error("agent exited with error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
module github.com/hexahost/gamecloud/node-agent
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/docker/docker v28.0.0+incompatible
|
||||
github.com/docker/go-connections v0.5.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
)
|
||||
|
||||
@@ -6,28 +6,43 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/config"
|
||||
dockerpkg "github.com/hexahost/gamecloud/node-agent/internal/docker"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/health"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/runtime"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/ws"
|
||||
)
|
||||
|
||||
const Version = "0.1.0-phase0"
|
||||
const Version = "0.2.0-phase2"
|
||||
|
||||
// Agent coordinates the node agent lifecycle until shutdown.
|
||||
type Agent struct {
|
||||
cfg *config.Config
|
||||
log *slog.Logger
|
||||
health *health.Server
|
||||
docker *dockerpkg.Client
|
||||
ws *ws.Client
|
||||
}
|
||||
|
||||
// New creates an agent instance from configuration.
|
||||
func New(cfg *config.Config, log *slog.Logger) *Agent {
|
||||
func New(cfg *config.Config, log *slog.Logger) (*Agent, error) {
|
||||
dockerClient, err := dockerpkg.New(cfg.DockerSocket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
wsClient := ws.NewClient(cfg, log, nil, Version)
|
||||
runtimeMgr := runtime.NewManager(dockerClient, wsClient, log)
|
||||
wsClient.SetRuntime(runtimeMgr)
|
||||
|
||||
return &Agent{
|
||||
cfg: cfg,
|
||||
log: log,
|
||||
health: health.NewServer(cfg.HealthListenAddr, Version),
|
||||
}
|
||||
docker: dockerClient,
|
||||
ws: wsClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Run starts the agent loop and blocks until context cancellation or signal.
|
||||
@@ -39,7 +54,7 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
defer signal.Stop(sigCh)
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
errCh := make(chan error, 2)
|
||||
go func() {
|
||||
a.log.Info("health server listening", "addr", a.cfg.HealthListenAddr)
|
||||
if err := a.health.ListenAndServe(); err != nil {
|
||||
@@ -47,6 +62,10 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
errCh <- a.ws.Run(ctx)
|
||||
}()
|
||||
|
||||
a.health.SetReady(true)
|
||||
a.log.Info("agent started",
|
||||
"node_id", a.cfg.NodeID,
|
||||
@@ -54,34 +73,33 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
"version", Version,
|
||||
)
|
||||
|
||||
ticker := time.NewTicker(a.cfg.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return a.shutdown(ctx.Err())
|
||||
return a.shutdown(nil)
|
||||
case sig := <-sigCh:
|
||||
a.log.Info("shutdown signal received", "signal", sig.String())
|
||||
return a.shutdown(nil)
|
||||
cancel()
|
||||
case err := <-errCh:
|
||||
return err
|
||||
case <-ticker.C:
|
||||
a.log.Debug("heartbeat tick",
|
||||
"node_id", a.cfg.NodeID,
|
||||
"interval", a.cfg.HeartbeatInterval.String(),
|
||||
)
|
||||
if ctx.Err() != nil {
|
||||
return a.shutdown(nil)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) shutdown(reason error) error {
|
||||
a.health.SetReady(false)
|
||||
a.log.Info("agent shutting down", "grace", a.cfg.ShutdownGracePeriod.String())
|
||||
a.log.Info("agent shutting down")
|
||||
|
||||
timer := time.NewTimer(a.cfg.ShutdownGracePeriod)
|
||||
defer timer.Stop()
|
||||
<-timer.C
|
||||
if a.docker != nil {
|
||||
if err := a.docker.Close(); err != nil {
|
||||
a.log.Warn("docker client close failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
if reason != nil && reason != context.Canceled {
|
||||
a.log.Warn("shutdown with error", "error", reason)
|
||||
|
||||
191
apps/node-agent/internal/docker/client.go
Normal file
191
apps/node-agent/internal/docker/client.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package docker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/docker/api/types/container"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
"github.com/docker/docker/api/types/mount"
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/docker/go-connections/nat"
|
||||
)
|
||||
|
||||
const MinecraftImage = "itzg/minecraft-server:java21"
|
||||
|
||||
// MinecraftServerSpec describes a Minecraft container to create.
|
||||
type MinecraftServerSpec struct {
|
||||
ServerID string
|
||||
MinecraftVersion string
|
||||
SoftwareFamily string
|
||||
HostPort int
|
||||
RamMb int
|
||||
DataPath string
|
||||
EulaAccepted bool
|
||||
}
|
||||
|
||||
// ContainerState summarizes a Docker container's runtime status.
|
||||
type ContainerState struct {
|
||||
ID string
|
||||
Status string
|
||||
Running bool
|
||||
}
|
||||
|
||||
// Client wraps the Docker Engine API for game server containers.
|
||||
type Client struct {
|
||||
cli *client.Client
|
||||
}
|
||||
|
||||
// New creates a Docker client connected via the given socket URL.
|
||||
func New(socket string) (*Client, error) {
|
||||
cli, err := client.NewClientWithOpts(
|
||||
client.FromEnv,
|
||||
client.WithHost(socket),
|
||||
client.WithAPIVersionNegotiation(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create docker client: %w", err)
|
||||
}
|
||||
return &Client{cli: cli}, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying Docker client.
|
||||
func (c *Client) Close() error {
|
||||
return c.cli.Close()
|
||||
}
|
||||
|
||||
// Ping verifies connectivity to the Docker daemon.
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
_, err := c.cli.Ping(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// CreateMinecraftServer creates a non-privileged Minecraft container and returns its ID.
|
||||
func (c *Client) CreateMinecraftServer(ctx context.Context, spec MinecraftServerSpec) (string, error) {
|
||||
if !spec.EulaAccepted {
|
||||
return "", fmt.Errorf("eula must be accepted before provisioning")
|
||||
}
|
||||
if spec.HostPort <= 0 {
|
||||
return "", fmt.Errorf("hostPort must be positive")
|
||||
}
|
||||
if strings.TrimSpace(spec.DataPath) == "" {
|
||||
return "", fmt.Errorf("dataPath is required")
|
||||
}
|
||||
if spec.RamMb <= 0 {
|
||||
return "", fmt.Errorf("ramMb must be positive")
|
||||
}
|
||||
|
||||
containerPort := nat.Port("25565/tcp")
|
||||
portBindings := nat.PortMap{
|
||||
containerPort: []nat.PortBinding{{
|
||||
HostIP: "0.0.0.0",
|
||||
HostPort: strconv.Itoa(spec.HostPort),
|
||||
}},
|
||||
}
|
||||
|
||||
version := strings.TrimSpace(spec.MinecraftVersion)
|
||||
if version == "" {
|
||||
version = "LATEST"
|
||||
}
|
||||
|
||||
resp, err := c.cli.ContainerCreate(ctx,
|
||||
&container.Config{
|
||||
Image: MinecraftImage,
|
||||
Env: []string{
|
||||
"EULA=TRUE",
|
||||
"VERSION=" + version,
|
||||
"TYPE=" + mapSoftwareFamily(spec.SoftwareFamily),
|
||||
fmt.Sprintf("MEMORY=%dM", spec.RamMb),
|
||||
},
|
||||
ExposedPorts: nat.PortSet{containerPort: struct{}{}},
|
||||
},
|
||||
&container.HostConfig{
|
||||
PortBindings: portBindings,
|
||||
Mounts: []mount.Mount{{
|
||||
Type: mount.TypeBind,
|
||||
Source: spec.DataPath,
|
||||
Target: "/data",
|
||||
}},
|
||||
Resources: container.Resources{
|
||||
Memory: int64(spec.RamMb) * 1024 * 1024,
|
||||
},
|
||||
Privileged: false,
|
||||
},
|
||||
nil,
|
||||
nil,
|
||||
containerName(spec.ServerID),
|
||||
)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create container: %w", err)
|
||||
}
|
||||
return resp.ID, nil
|
||||
}
|
||||
|
||||
// StartContainer starts a container by ID.
|
||||
func (c *Client) StartContainer(ctx context.Context, containerID string) error {
|
||||
if err := c.cli.ContainerStart(ctx, containerID, container.StartOptions{}); err != nil {
|
||||
return fmt.Errorf("start container: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StopContainer stops a running container gracefully.
|
||||
func (c *Client) StopContainer(ctx context.Context, containerID string) error {
|
||||
timeout := 30
|
||||
if err := c.cli.ContainerStop(ctx, containerID, container.StopOptions{Timeout: &timeout}); err != nil {
|
||||
return fmt.Errorf("stop container: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveContainer removes a container, forcing removal if still running.
|
||||
func (c *Client) RemoveContainer(ctx context.Context, containerID string) error {
|
||||
if err := c.cli.ContainerRemove(ctx, containerID, container.RemoveOptions{Force: true}); err != nil {
|
||||
return fmt.Errorf("remove container: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetContainerState inspects a container and returns its current state.
|
||||
func (c *Client) GetContainerState(ctx context.Context, containerID string) (*ContainerState, error) {
|
||||
inspect, err := c.cli.ContainerInspect(ctx, containerID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect container: %w", err)
|
||||
}
|
||||
return &ContainerState{
|
||||
ID: inspect.ID,
|
||||
Status: inspect.State.Status,
|
||||
Running: inspect.State.Running,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ListManagedContainers returns container IDs labeled for a given server ID.
|
||||
func (c *Client) ListManagedContainers(ctx context.Context, serverID string) ([]string, error) {
|
||||
containers, err := c.cli.ContainerList(ctx, container.ListOptions{
|
||||
All: true,
|
||||
Filters: filters.NewArgs(filters.Arg("name", containerName(serverID))),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list containers: %w", err)
|
||||
}
|
||||
ids := make([]string, 0, len(containers))
|
||||
for _, ctr := range containers {
|
||||
ids = append(ids, ctr.ID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func containerName(serverID string) string {
|
||||
return "hexahost-" + serverID
|
||||
}
|
||||
|
||||
func mapSoftwareFamily(family string) string {
|
||||
switch strings.ToUpper(strings.TrimSpace(family)) {
|
||||
case "PAPER":
|
||||
return "PAPER"
|
||||
default:
|
||||
return "VANILLA"
|
||||
}
|
||||
}
|
||||
@@ -120,6 +120,31 @@ type ServerStartPayload struct {
|
||||
DesiredGeneration int64 `json:"desiredGeneration"`
|
||||
}
|
||||
|
||||
// ServerProvisionPayload instructs the agent to create a Minecraft server container.
|
||||
type ServerProvisionPayload struct {
|
||||
OperationMeta
|
||||
Edition string `json:"edition"`
|
||||
SoftwareFamily string `json:"softwareFamily"`
|
||||
MinecraftVersion string `json:"minecraftVersion"`
|
||||
HostPort int `json:"hostPort"`
|
||||
RamMb int `json:"ramMb"`
|
||||
DataPath string `json:"dataPath"`
|
||||
EulaAccepted bool `json:"eulaAccepted"`
|
||||
}
|
||||
|
||||
// ServerStopPayload instructs the agent to stop a running game server.
|
||||
type ServerStopPayload struct {
|
||||
OperationMeta
|
||||
}
|
||||
|
||||
// ServerOperationResultPayload reports the outcome of a lifecycle operation.
|
||||
type ServerOperationResultPayload struct {
|
||||
OperationMeta
|
||||
ResultCode string `json:"resultCode"`
|
||||
ErrorCode string `json:"errorCode,omitempty"`
|
||||
ErrorMessage string `json:"errorMessage,omitempty"`
|
||||
}
|
||||
|
||||
// ServerStatePayload reports the observed server state.
|
||||
type ServerStatePayload struct {
|
||||
OperationMeta
|
||||
|
||||
@@ -136,6 +136,69 @@ func TestDecodeEnvelope(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerProvisionPayload_JSONRoundtrip(t *testing.T) {
|
||||
deadline := time.Date(2026, 6, 26, 12, 0, 0, 0, time.UTC)
|
||||
original := ServerProvisionPayload{
|
||||
OperationMeta: OperationMeta{
|
||||
ServerID: "srv-abc123",
|
||||
Generation: 2,
|
||||
Deadline: &deadline,
|
||||
IdempotencyKey: "idem-001",
|
||||
},
|
||||
Edition: "JAVA",
|
||||
SoftwareFamily: "PAPER",
|
||||
MinecraftVersion: "1.21.1",
|
||||
HostPort: 25565,
|
||||
RamMb: 2048,
|
||||
DataPath: "/var/lib/hexahost-gamecloud/srv-abc123",
|
||||
EulaAccepted: true,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
var decoded ServerProvisionPayload
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
if decoded.ServerID != original.ServerID {
|
||||
t.Fatalf("serverId: got %q want %q", decoded.ServerID, original.ServerID)
|
||||
}
|
||||
if decoded.Generation != original.Generation {
|
||||
t.Fatalf("generation: got %d want %d", decoded.Generation, original.Generation)
|
||||
}
|
||||
if decoded.Edition != original.Edition {
|
||||
t.Fatalf("edition: got %q want %q", decoded.Edition, original.Edition)
|
||||
}
|
||||
if decoded.SoftwareFamily != original.SoftwareFamily {
|
||||
t.Fatalf("softwareFamily: got %q want %q", decoded.SoftwareFamily, original.SoftwareFamily)
|
||||
}
|
||||
if decoded.MinecraftVersion != original.MinecraftVersion {
|
||||
t.Fatalf("minecraftVersion: got %q want %q", decoded.MinecraftVersion, original.MinecraftVersion)
|
||||
}
|
||||
if decoded.HostPort != original.HostPort {
|
||||
t.Fatalf("hostPort: got %d want %d", decoded.HostPort, original.HostPort)
|
||||
}
|
||||
if decoded.RamMb != original.RamMb {
|
||||
t.Fatalf("ramMb: got %d want %d", decoded.RamMb, original.RamMb)
|
||||
}
|
||||
if decoded.DataPath != original.DataPath {
|
||||
t.Fatalf("dataPath: got %q want %q", decoded.DataPath, original.DataPath)
|
||||
}
|
||||
if decoded.EulaAccepted != original.EulaAccepted {
|
||||
t.Fatalf("eulaAccepted: got %v want %v", decoded.EulaAccepted, original.EulaAccepted)
|
||||
}
|
||||
if decoded.IdempotencyKey != original.IdempotencyKey {
|
||||
t.Fatalf("idempotencyKey: got %q want %q", decoded.IdempotencyKey, original.IdempotencyKey)
|
||||
}
|
||||
if decoded.Deadline == nil || !decoded.Deadline.Equal(deadline) {
|
||||
t.Fatalf("deadline mismatch: got %v want %v", decoded.Deadline, deadline)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEnvelope(t *testing.T) {
|
||||
env, err := NewEnvelope(TypeAgentHello, "msg-new", AgentHelloPayload{
|
||||
NodeID: "n1",
|
||||
|
||||
269
apps/node-agent/internal/runtime/manager.go
Normal file
269
apps/node-agent/internal/runtime/manager.go
Normal file
@@ -0,0 +1,269 @@
|
||||
package runtime
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/docker"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/protocol"
|
||||
)
|
||||
|
||||
const (
|
||||
ServerStateProvisioning = "provisioning"
|
||||
ServerStateStopped = "stopped"
|
||||
ServerStateStarting = "starting"
|
||||
ServerStateRunning = "running"
|
||||
ServerStateStopping = "stopping"
|
||||
ServerStateDeleting = "deleting"
|
||||
ServerStateDeleted = "deleted"
|
||||
ServerStateError = "error"
|
||||
|
||||
ResultCodeOK = "ok"
|
||||
ResultCodeError = "error"
|
||||
)
|
||||
|
||||
// Responder sends protocol responses back to the control plane.
|
||||
type Responder interface {
|
||||
SendState(ctx context.Context, correlationID string, payload protocol.ServerStatePayload) error
|
||||
SendOperationComplete(ctx context.Context, correlationID string, payload protocol.ServerOperationResultPayload) error
|
||||
SendOperationFailed(ctx context.Context, correlationID string, payload protocol.ServerOperationResultPayload) error
|
||||
}
|
||||
|
||||
type serverRecord struct {
|
||||
ServerID string
|
||||
ContainerID string
|
||||
HostPort int
|
||||
DataPath string
|
||||
RamMb int
|
||||
State string
|
||||
}
|
||||
|
||||
// Manager tracks provisioned servers and executes lifecycle operations.
|
||||
type Manager struct {
|
||||
mu sync.RWMutex
|
||||
docker *docker.Client
|
||||
servers map[string]*serverRecord
|
||||
resp Responder
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// NewManager creates a runtime manager backed by the given Docker client.
|
||||
func NewManager(dockerClient *docker.Client, resp Responder, log *slog.Logger) *Manager {
|
||||
return &Manager{
|
||||
docker: dockerClient,
|
||||
servers: make(map[string]*serverRecord),
|
||||
resp: resp,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// RunningCount returns the number of servers currently in the running state.
|
||||
func (m *Manager) RunningCount() int {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
count := 0
|
||||
for _, rec := range m.servers {
|
||||
if rec.State == ServerStateRunning {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// HandleProvision creates a Minecraft container for the server.
|
||||
func (m *Manager) HandleProvision(ctx context.Context, correlationID string, payload protocol.ServerProvisionPayload) {
|
||||
meta := payload.OperationMeta
|
||||
m.log.Info("provision server",
|
||||
"server_id", meta.ServerID,
|
||||
"host_port", payload.HostPort,
|
||||
"version", payload.MinecraftVersion,
|
||||
)
|
||||
|
||||
if !payload.EulaAccepted {
|
||||
m.fail(ctx, correlationID, meta, "EULA_NOT_ACCEPTED", "EULA must be accepted before provisioning")
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if existing, ok := m.servers[meta.ServerID]; ok {
|
||||
m.mu.Unlock()
|
||||
m.log.Info("provision idempotent hit", "server_id", meta.ServerID, "container_id", existing.ContainerID)
|
||||
m.complete(ctx, correlationID, meta, existing.State)
|
||||
return
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
m.emitState(ctx, correlationID, meta, ServerStateProvisioning, "", "")
|
||||
|
||||
containerID, err := m.docker.CreateMinecraftServer(ctx, docker.MinecraftServerSpec{
|
||||
ServerID: meta.ServerID,
|
||||
MinecraftVersion: payload.MinecraftVersion,
|
||||
SoftwareFamily: payload.SoftwareFamily,
|
||||
HostPort: payload.HostPort,
|
||||
RamMb: payload.RamMb,
|
||||
DataPath: payload.DataPath,
|
||||
EulaAccepted: payload.EulaAccepted,
|
||||
})
|
||||
if err != nil {
|
||||
m.fail(ctx, correlationID, meta, "PROVISION_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
rec := &serverRecord{
|
||||
ServerID: meta.ServerID,
|
||||
ContainerID: containerID,
|
||||
HostPort: payload.HostPort,
|
||||
DataPath: payload.DataPath,
|
||||
RamMb: payload.RamMb,
|
||||
State: ServerStateStopped,
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.servers[meta.ServerID] = rec
|
||||
m.mu.Unlock()
|
||||
|
||||
m.emitState(ctx, correlationID, meta, ServerStateStopped, "", "")
|
||||
m.complete(ctx, correlationID, meta, ServerStateStopped)
|
||||
}
|
||||
|
||||
// HandleStart starts a provisioned server container.
|
||||
func (m *Manager) HandleStart(ctx context.Context, correlationID string, payload protocol.ServerStartPayload) {
|
||||
meta := payload.OperationMeta
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.fail(ctx, correlationID, meta, "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
m.emitState(ctx, correlationID, meta, ServerStateStarting, "", "")
|
||||
|
||||
if err := m.docker.StartContainer(ctx, rec.ContainerID); err != nil {
|
||||
m.setState(meta.ServerID, ServerStateError)
|
||||
m.emitState(ctx, correlationID, meta, ServerStateError, "START_FAILED", err.Error())
|
||||
m.fail(ctx, correlationID, meta, "START_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.setState(meta.ServerID, ServerStateRunning)
|
||||
m.emitState(ctx, correlationID, meta, ServerStateRunning, "", "")
|
||||
m.complete(ctx, correlationID, meta, ServerStateRunning)
|
||||
}
|
||||
|
||||
// HandleStop stops a running server container.
|
||||
func (m *Manager) HandleStop(ctx context.Context, correlationID string, payload protocol.ServerStopPayload) {
|
||||
meta := payload.OperationMeta
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.fail(ctx, correlationID, meta, "SERVER_NOT_FOUND", "server is not provisioned on this node")
|
||||
return
|
||||
}
|
||||
|
||||
m.emitState(ctx, correlationID, meta, ServerStateStopping, "", "")
|
||||
|
||||
if err := m.docker.StopContainer(ctx, rec.ContainerID); err != nil {
|
||||
m.setState(meta.ServerID, ServerStateError)
|
||||
m.emitState(ctx, correlationID, meta, ServerStateError, "STOP_FAILED", err.Error())
|
||||
m.fail(ctx, correlationID, meta, "STOP_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.setState(meta.ServerID, ServerStateStopped)
|
||||
m.emitState(ctx, correlationID, meta, ServerStateStopped, "", "")
|
||||
m.complete(ctx, correlationID, meta, ServerStateStopped)
|
||||
}
|
||||
|
||||
// HandleDelete removes a server container and drops local state.
|
||||
func (m *Manager) HandleDelete(ctx context.Context, correlationID string, meta protocol.OperationMeta) {
|
||||
rec, ok := m.getServer(meta.ServerID)
|
||||
if !ok {
|
||||
m.complete(ctx, correlationID, meta, ServerStateDeleted)
|
||||
return
|
||||
}
|
||||
|
||||
m.emitState(ctx, correlationID, meta, ServerStateDeleting, "", "")
|
||||
|
||||
if state, err := m.docker.GetContainerState(ctx, rec.ContainerID); err == nil && state.Running {
|
||||
if err := m.docker.StopContainer(ctx, rec.ContainerID); err != nil {
|
||||
m.fail(ctx, correlationID, meta, "DELETE_STOP_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := m.docker.RemoveContainer(ctx, rec.ContainerID); err != nil {
|
||||
m.fail(ctx, correlationID, meta, "DELETE_FAILED", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
delete(m.servers, meta.ServerID)
|
||||
m.mu.Unlock()
|
||||
|
||||
m.emitState(ctx, correlationID, meta, ServerStateDeleted, "", "")
|
||||
m.complete(ctx, correlationID, meta, ServerStateDeleted)
|
||||
}
|
||||
|
||||
func (m *Manager) getServer(serverID string) (*serverRecord, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
rec, ok := m.servers[serverID]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
copy := *rec
|
||||
return ©, true
|
||||
}
|
||||
|
||||
func (m *Manager) setState(serverID, state string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if rec, ok := m.servers[serverID]; ok {
|
||||
rec.State = state
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) emitState(ctx context.Context, correlationID string, meta protocol.OperationMeta, state, errorCode, errorMessage string) {
|
||||
payload := protocol.ServerStatePayload{
|
||||
OperationMeta: meta,
|
||||
State: state,
|
||||
ErrorCode: errorCode,
|
||||
ErrorMessage: errorMessage,
|
||||
}
|
||||
if err := m.resp.SendState(ctx, correlationID, payload); err != nil {
|
||||
m.log.Warn("send server.state failed",
|
||||
"server_id", meta.ServerID,
|
||||
"state", state,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) complete(ctx context.Context, correlationID string, meta protocol.OperationMeta, _ string) {
|
||||
payload := protocol.ServerOperationResultPayload{
|
||||
OperationMeta: meta,
|
||||
ResultCode: ResultCodeOK,
|
||||
}
|
||||
if err := m.resp.SendOperationComplete(ctx, correlationID, payload); err != nil {
|
||||
m.log.Warn("send operation complete failed",
|
||||
"server_id", meta.ServerID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) fail(ctx context.Context, correlationID string, meta protocol.OperationMeta, errorCode, errorMessage string) {
|
||||
m.emitState(ctx, correlationID, meta, ServerStateError, errorCode, errorMessage)
|
||||
payload := protocol.ServerOperationResultPayload{
|
||||
OperationMeta: meta,
|
||||
ResultCode: ResultCodeError,
|
||||
ErrorCode: errorCode,
|
||||
ErrorMessage: errorMessage,
|
||||
}
|
||||
if err := m.resp.SendOperationFailed(ctx, correlationID, payload); err != nil {
|
||||
m.log.Warn("send operation failed failed",
|
||||
"server_id", meta.ServerID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
}
|
||||
357
apps/node-agent/internal/ws/client.go
Normal file
357
apps/node-agent/internal/ws/client.go
Normal file
@@ -0,0 +1,357 @@
|
||||
package ws
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/config"
|
||||
"github.com/hexahost/gamecloud/node-agent/internal/protocol"
|
||||
runtimepkg "github.com/hexahost/gamecloud/node-agent/internal/runtime"
|
||||
)
|
||||
|
||||
const (
|
||||
writeWait = 10 * time.Second
|
||||
pongWait = 60 * time.Second
|
||||
pingPeriod = (pongWait * 9) / 10
|
||||
maxReconnectDelay = 60 * time.Second
|
||||
)
|
||||
|
||||
// Client maintains an outbound WebSocket connection to the control plane.
|
||||
type Client struct {
|
||||
cfg *config.Config
|
||||
log *slog.Logger
|
||||
runtime *runtimepkg.Manager
|
||||
agentVersion string
|
||||
|
||||
writeMu sync.Mutex
|
||||
conn *websocket.Conn
|
||||
}
|
||||
|
||||
// NewClient creates a WebSocket client for the control plane.
|
||||
func NewClient(cfg *config.Config, log *slog.Logger, mgr *runtimepkg.Manager, agentVersion string) *Client {
|
||||
return &Client{
|
||||
cfg: cfg,
|
||||
log: log,
|
||||
runtime: mgr,
|
||||
agentVersion: agentVersion,
|
||||
}
|
||||
}
|
||||
|
||||
// SetRuntime attaches the runtime manager after construction to break init cycles.
|
||||
func (c *Client) SetRuntime(mgr *runtimepkg.Manager) {
|
||||
c.runtime = mgr
|
||||
}
|
||||
|
||||
// Run connects to the control plane and processes messages until ctx is cancelled.
|
||||
func (c *Client) Run(ctx context.Context) error {
|
||||
backoff := c.cfg.ReconnectBackoff
|
||||
if backoff <= 0 {
|
||||
backoff = 5 * time.Second
|
||||
}
|
||||
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
err := c.session(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
c.log.Warn("websocket disconnected", "error", err, "retry_in", backoff.String())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(backoff):
|
||||
}
|
||||
|
||||
backoff *= 2
|
||||
if backoff > maxReconnectDelay {
|
||||
backoff = maxReconnectDelay
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) session(ctx context.Context) error {
|
||||
conn, err := c.dial(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.writeMu.Lock()
|
||||
c.conn = conn
|
||||
c.writeMu.Unlock()
|
||||
|
||||
defer func() {
|
||||
c.writeMu.Lock()
|
||||
c.conn = nil
|
||||
c.writeMu.Unlock()
|
||||
_ = conn.Close()
|
||||
}()
|
||||
|
||||
conn.SetReadDeadline(time.Now().Add(pongWait))
|
||||
conn.SetPongHandler(func(string) error {
|
||||
return conn.SetReadDeadline(time.Now().Add(pongWait))
|
||||
})
|
||||
|
||||
if err := c.sendHello(ctx); err != nil {
|
||||
return fmt.Errorf("send hello: %w", err)
|
||||
}
|
||||
|
||||
sessionCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
errCh := make(chan error, 2)
|
||||
go func() {
|
||||
errCh <- c.heartbeatLoop(sessionCtx)
|
||||
}()
|
||||
go func() {
|
||||
errCh <- c.readLoop(sessionCtx)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
cancel()
|
||||
return ctx.Err()
|
||||
case err := <-errCh:
|
||||
cancel()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) dial(ctx context.Context) (*websocket.Conn, error) {
|
||||
endpoint, err := c.buildURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dialer := websocket.DefaultDialer
|
||||
conn, _, err := dialer.DialContext(ctx, endpoint, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dial websocket: %w", err)
|
||||
}
|
||||
c.log.Info("connected to control plane", "url", redactToken(endpoint))
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (c *Client) buildURL() (string, error) {
|
||||
u, err := url.Parse(c.cfg.ControlPlaneURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse control plane url: %w", err)
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("nodeId", c.cfg.NodeID)
|
||||
if c.cfg.NodeToken != "" {
|
||||
q.Set("token", c.cfg.NodeToken)
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func (c *Client) sendHello(ctx context.Context) error {
|
||||
hostname, _ := os.Hostname()
|
||||
payload := protocol.AgentHelloPayload{
|
||||
NodeID: c.cfg.NodeID,
|
||||
AgentVersion: c.agentVersion,
|
||||
ProtocolVersion: protocol.CurrentProtocolVersion,
|
||||
Hostname: hostname,
|
||||
Capabilities: []string{"minecraft", "docker"},
|
||||
}
|
||||
return c.send(ctx, protocol.TypeAgentHello, "", payload)
|
||||
}
|
||||
|
||||
func (c *Client) heartbeatLoop(ctx context.Context) error {
|
||||
ticker := time.NewTicker(c.cfg.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
pingTicker := time.NewTicker(pingPeriod)
|
||||
defer pingTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if err := c.sendHeartbeat(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
case <-pingTicker.C:
|
||||
if err := c.writePing(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) sendHeartbeat(ctx context.Context) error {
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
|
||||
payload := protocol.AgentHeartbeatPayload{
|
||||
NodeID: c.cfg.NodeID,
|
||||
CPUUsagePercent: 0,
|
||||
MemoryUsedBytes: int64(memStats.Alloc),
|
||||
MemoryTotalBytes: int64(memStats.Sys),
|
||||
RunningServers: c.runtime.RunningCount(),
|
||||
}
|
||||
return c.send(ctx, protocol.TypeAgentHeartbeat, "", payload)
|
||||
}
|
||||
|
||||
func (c *Client) readLoop(ctx context.Context) error {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
c.writeMu.Lock()
|
||||
conn := c.conn
|
||||
c.writeMu.Unlock()
|
||||
if conn == nil {
|
||||
return fmt.Errorf("connection closed")
|
||||
}
|
||||
|
||||
_, data, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read message: %w", err)
|
||||
}
|
||||
|
||||
env, err := protocol.DecodeEnvelope(data)
|
||||
if err != nil {
|
||||
c.log.Warn("invalid inbound envelope", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
c.dispatch(ctx, env)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) dispatch(ctx context.Context, env *protocol.Envelope) {
|
||||
correlationID := env.CorrelationID
|
||||
if correlationID == "" {
|
||||
correlationID = env.MessageID
|
||||
}
|
||||
|
||||
switch env.Type {
|
||||
case protocol.TypeServerProvision:
|
||||
var payload protocol.ServerProvisionPayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.provision", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleProvision(ctx, correlationID, payload)
|
||||
|
||||
case protocol.TypeServerStart:
|
||||
var payload protocol.ServerStartPayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.start", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleStart(ctx, correlationID, payload)
|
||||
|
||||
case protocol.TypeServerStop:
|
||||
var payload protocol.ServerStopPayload
|
||||
if err := json.Unmarshal(env.Payload, &payload); err != nil {
|
||||
c.log.Warn("decode server.stop", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleStop(ctx, correlationID, payload)
|
||||
|
||||
case protocol.TypeServerDelete:
|
||||
var meta protocol.OperationMeta
|
||||
if err := json.Unmarshal(env.Payload, &meta); err != nil {
|
||||
c.log.Warn("decode server.delete", "error", err)
|
||||
return
|
||||
}
|
||||
c.runtime.HandleDelete(ctx, correlationID, meta)
|
||||
|
||||
default:
|
||||
c.log.Debug("ignored control message", "type", env.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) send(ctx context.Context, msgType protocol.MessageType, correlationID string, payload any) error {
|
||||
env, err := protocol.NewEnvelope(msgType, uuid.NewString(), payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if correlationID != "" {
|
||||
env.CorrelationID = correlationID
|
||||
}
|
||||
return c.writeEnvelope(ctx, env)
|
||||
}
|
||||
|
||||
func (c *Client) writeEnvelope(ctx context.Context, env *protocol.Envelope) error {
|
||||
data, err := json.Marshal(env)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal envelope: %w", err)
|
||||
}
|
||||
|
||||
c.writeMu.Lock()
|
||||
conn := c.conn
|
||||
c.writeMu.Unlock()
|
||||
if conn == nil {
|
||||
return fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
deadline, ok := ctx.Deadline()
|
||||
if !ok {
|
||||
deadline = time.Now().Add(writeWait)
|
||||
}
|
||||
if err := conn.SetWriteDeadline(deadline); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
if c.conn == nil {
|
||||
return fmt.Errorf("not connected")
|
||||
}
|
||||
return c.conn.WriteMessage(websocket.TextMessage, data)
|
||||
}
|
||||
|
||||
func (c *Client) writePing() error {
|
||||
c.writeMu.Lock()
|
||||
defer c.writeMu.Unlock()
|
||||
if c.conn == nil {
|
||||
return fmt.Errorf("not connected")
|
||||
}
|
||||
return c.conn.WriteMessage(websocket.PingMessage, nil)
|
||||
}
|
||||
|
||||
// SendState implements runtime.Responder.
|
||||
func (c *Client) SendState(ctx context.Context, correlationID string, payload protocol.ServerStatePayload) error {
|
||||
return c.send(ctx, protocol.TypeServerState, correlationID, payload)
|
||||
}
|
||||
|
||||
// SendOperationComplete implements runtime.Responder.
|
||||
func (c *Client) SendOperationComplete(ctx context.Context, correlationID string, payload protocol.ServerOperationResultPayload) error {
|
||||
return c.send(ctx, protocol.TypeServerOperationComplete, correlationID, payload)
|
||||
}
|
||||
|
||||
// SendOperationFailed implements runtime.Responder.
|
||||
func (c *Client) SendOperationFailed(ctx context.Context, correlationID string, payload protocol.ServerOperationResultPayload) error {
|
||||
return c.send(ctx, protocol.TypeServerOperationFailed, correlationID, payload)
|
||||
}
|
||||
|
||||
func redactToken(rawURL string) string {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return rawURL
|
||||
}
|
||||
if u.Query().Get("token") != "" {
|
||||
q := u.Query()
|
||||
q.Set("token", "REDACTED")
|
||||
u.RawQuery = q.Encode()
|
||||
}
|
||||
return u.String()
|
||||
}
|
||||
Reference in New Issue
Block a user