Phase F
Some checks failed
CI / Node — lint, typecheck, test, build (push) Failing after 10s
CI / Go — node-agent tests (push) Failing after 8s
CI / Go — edge-gateway build (push) Successful in 18s

This commit is contained in:
smueller
2026-06-26 15:05:01 +02:00
parent 333ad1cc7d
commit 15cf302afc
23 changed files with 2116 additions and 8 deletions

View File

@@ -0,0 +1,148 @@
import { Injectable, NotFoundException } from '@nestjs/common';
import type {
WhmcsEventsAckRequest,
WhmcsEventsAckResponse,
WhmcsEventsResponse,
} from '@hexahost/contracts';
import type { Prisma } from '@hexahost/database';
import { PrismaService } from '../../prisma/prisma.service';
type InstallationRef = {
id: string;
};
@Injectable()
export class WhmcsEventsService {
constructor(private readonly prisma: PrismaService) {}
async listEvents(
installation: InstallationRef,
cursor?: string,
limit = 50,
): Promise<WhmcsEventsResponse> {
const take = Math.min(Math.max(limit, 1), 100);
const events = await this.prisma.integrationEvent.findMany({
where: {
installationId: installation.id,
status: 'PENDING',
...(cursor
? {
createdAt: {
gt: await this.cursorCreatedAt(installation.id, cursor),
},
}
: {}),
},
orderBy: { createdAt: 'asc' },
take: take + 1,
});
const page = events.slice(0, take);
const nextCursor =
events.length > take ? (page[page.length - 1]?.id ?? null) : null;
return {
events: page.map((event) => ({
id: event.id,
eventType: event.eventType,
payload: event.payload as Record<string, unknown>,
status: event.status,
createdAt: event.createdAt.toISOString(),
})),
nextCursor,
};
}
async acknowledgeEvents(
installation: InstallationRef,
input: WhmcsEventsAckRequest,
): Promise<WhmcsEventsAckResponse> {
const events = await this.prisma.integrationEvent.findMany({
where: {
installationId: installation.id,
id: { in: input.eventIds },
},
});
if (events.length !== input.eventIds.length) {
throw new NotFoundException('One or more events were not found');
}
const now = new Date();
if (input.deadLetter) {
await this.prisma.integrationEvent.updateMany({
where: {
installationId: installation.id,
id: { in: input.eventIds },
},
data: {
status: 'DEAD_LETTER',
deadLetteredAt: now,
deadLetterReason: input.deadLetterReason ?? 'Marked dead letter by WHMCS',
},
});
} else {
await this.prisma.integrationEvent.updateMany({
where: {
installationId: installation.id,
id: { in: input.eventIds },
},
data: {
status: 'ACKNOWLEDGED',
acknowledgedAt: now,
},
});
const lastId = input.eventIds[input.eventIds.length - 1];
if (lastId) {
await this.prisma.integrationSyncCursor.upsert({
where: { installationId: installation.id },
create: {
installationId: installation.id,
eventsCursor: lastId,
},
update: {
eventsCursor: lastId,
},
});
}
}
return { acknowledged: events.length };
}
async emitEvent(
installationId: string,
eventType: string,
payload: Record<string, unknown>,
dedupeKey: string,
): Promise<void> {
try {
await this.prisma.integrationEvent.create({
data: {
installationId,
eventType,
payload: payload as Prisma.InputJsonValue,
dedupeKey,
},
});
} catch {
// Duplicate dedupe key — event already emitted
}
}
private async cursorCreatedAt(
installationId: string,
cursor: string,
): Promise<Date> {
const event = await this.prisma.integrationEvent.findFirst({
where: { id: cursor, installationId },
});
return event?.createdAt ?? new Date(0);
}
}

View File

@@ -6,6 +6,7 @@ import {
Patch,
Post,
Put,
Query,
Req,
UseGuards,
} from '@nestjs/common';
@@ -14,9 +15,14 @@ import { SkipThrottle } from '@nestjs/throttler';
import {
whmcsChangePackageRequestSchema,
whmcsCreateSsoRequestSchema,
whmcsEventsAckRequestSchema,
whmcsImportServiceRequestSchema,
whmcsProvisionServiceRequestSchema,
whmcsReconcileRequestSchema,
whmcsSuspendRequestSchema,
whmcsUpsertClientRequestSchema,
whmcsUsageAdjustmentRequestSchema,
whmcsUsageQuerySchema,
} from '@hexahost/contracts';
import { INTEGRATION_HEADERS } from '@hexahost/integration-auth';
@@ -27,6 +33,9 @@ import {
type WhmcsAuthenticatedRequest,
} from './whmcs-integration.guard';
import { WhmcsIntegrationService } from './whmcs-integration.service';
import { WhmcsEventsService } from './whmcs-events.service';
import { WhmcsReconciliationService } from './whmcs-reconciliation.service';
import { WhmcsUsageService } from './whmcs-usage.service';
@Controller('integrations/whmcs')
@UseGuards(WhmcsIntegrationGuard)
@@ -35,6 +44,9 @@ export class WhmcsIntegrationController {
constructor(
private readonly whmcs: WhmcsIntegrationService,
private readonly sso: SsoService,
private readonly reconciliation: WhmcsReconciliationService,
private readonly events: WhmcsEventsService,
private readonly usage: WhmcsUsageService,
) {}
@Get('health')
@@ -148,6 +160,95 @@ export class WhmcsIntegrationController {
headerValue(request, INTEGRATION_HEADERS.idempotencyKey)!,
);
}
@Get('accounts')
listAccounts(@Req() request: WhmcsAuthenticatedRequest) {
return this.reconciliation.listAccounts(request.whmcsInstallation!);
}
@Post('services/import')
importService(@Req() request: WhmcsAuthenticatedRequest, @Body() body: unknown) {
const parsed = whmcsImportServiceRequestSchema.parse(body);
return this.reconciliation.importService(
request.whmcsInstallation!,
parsed,
headerValue(request, INTEGRATION_HEADERS.idempotencyKey)!,
);
}
@Post('reconcile')
reconcileAll(@Req() request: WhmcsAuthenticatedRequest, @Body() body: unknown) {
const parsed = whmcsReconcileRequestSchema.parse(body ?? {});
return this.reconciliation.reconcileInstallation(
request.whmcsInstallation!,
parsed,
);
}
@Post('services/:externalServiceId/reconcile')
reconcileService(
@Req() request: WhmcsAuthenticatedRequest,
@Param('externalServiceId') externalServiceId: string,
@Body() body: unknown,
) {
const parsed = whmcsReconcileRequestSchema.parse(body ?? {});
return this.reconciliation.reconcileService(
request.whmcsInstallation!,
externalServiceId,
parsed,
);
}
@Get('events')
listEvents(
@Req() request: WhmcsAuthenticatedRequest,
@Query('cursor') cursor?: string,
@Query('limit') limit?: string,
) {
const parsedLimit = limit ? Number.parseInt(limit, 10) : 50;
return this.events.listEvents(
request.whmcsInstallation!,
cursor,
Number.isFinite(parsedLimit) ? parsedLimit : 50,
);
}
@Post('events/ack')
acknowledgeEvents(@Req() request: WhmcsAuthenticatedRequest, @Body() body: unknown) {
const parsed = whmcsEventsAckRequestSchema.parse(body);
return this.events.acknowledgeEvents(request.whmcsInstallation!, parsed);
}
@Get('services/:externalServiceId/usage')
getServiceUsage(
@Req() request: WhmcsAuthenticatedRequest,
@Param('externalServiceId') externalServiceId: string,
@Query() query: unknown,
) {
const parsed = whmcsUsageQuerySchema.parse(query);
return this.usage.getServiceUsage(
request.whmcsInstallation!,
externalServiceId,
parsed.periodStart,
parsed.periodEnd,
parsed.testMode,
);
}
@Post('services/:externalServiceId/usage/adjustments')
createUsageAdjustment(
@Req() request: WhmcsAuthenticatedRequest,
@Param('externalServiceId') externalServiceId: string,
@Body() body: unknown,
) {
const parsed = whmcsUsageAdjustmentRequestSchema.parse(body);
return this.usage.createAdjustment(
request.whmcsInstallation!,
externalServiceId,
parsed,
headerValue(request, INTEGRATION_HEADERS.idempotencyKey)!,
);
}
}
function headerValue(request: WhmcsAuthenticatedRequest, header: string): string | undefined {

View File

@@ -8,9 +8,12 @@ import { AuthModule } from '../../auth/auth.module';
import { ServersModule } from '../../servers/servers.module';
import { WhmcsIntegrationController } from './whmcs-integration.controller';
import { WhmcsEventsService } from './whmcs-events.service';
import { INTEGRATIONS_REDIS } from './whmcs-integration.constants';
import { WhmcsIntegrationGuard } from './whmcs-integration.guard';
import { WhmcsIntegrationService } from './whmcs-integration.service';
import { WhmcsReconciliationService } from './whmcs-reconciliation.service';
import { WhmcsUsageService } from './whmcs-usage.service';
export { INTEGRATIONS_REDIS } from './whmcs-integration.constants';
@@ -19,6 +22,9 @@ export { INTEGRATIONS_REDIS } from './whmcs-integration.constants';
controllers: [WhmcsIntegrationController],
providers: [
WhmcsIntegrationService,
WhmcsReconciliationService,
WhmcsEventsService,
WhmcsUsageService,
WhmcsIntegrationGuard,
{
provide: INTEGRATIONS_REDIS,

View File

@@ -0,0 +1,24 @@
export function resolveBillingPeriod(
periodStart?: string,
periodEnd?: string,
): { start: Date; end: Date } {
if (periodStart && periodEnd) {
return {
start: new Date(periodStart),
end: new Date(periodEnd),
};
}
const now = new Date();
const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
return { start, end };
}
export function previousBillingPeriod(reference = new Date()): { start: Date; end: Date } {
const start = new Date(
Date.UTC(reference.getUTCFullYear(), reference.getUTCMonth() - 1, 1),
);
const end = new Date(Date.UTC(reference.getUTCFullYear(), reference.getUTCMonth(), 1));
return { start, end };
}

View File

@@ -0,0 +1,573 @@
import { randomUUID } from 'node:crypto';
import {
ConflictException,
Injectable,
NotFoundException,
} from '@nestjs/common';
import type {
WhmcsImportServiceRequest,
WhmcsListAccountsResponse,
WhmcsReconcileRequest,
WhmcsReconcileResponse,
} from '@hexahost/contracts';
import { formatJoinAddress } from '@hexahost/dns';
import type { Prisma } from '@hexahost/database';
import { PrismaService } from '../../prisma/prisma.service';
import { WhmcsEventsService } from './whmcs-events.service';
type InstallationRef = {
id: string;
integrationId: string;
};
type ReconciliationSeverity =
| 'INFO'
| 'WARNING'
| 'BLOCKING'
| 'DESTRUCTIVE'
| 'SECURITY';
type IssueDraft = {
code: string;
severity: ReconciliationSeverity;
message: string;
externalServiceId?: string;
serverId?: string;
details?: Record<string, unknown>;
};
@Injectable()
export class WhmcsReconciliationService {
constructor(
private readonly prisma: PrismaService,
private readonly events: WhmcsEventsService,
) {}
async listAccounts(installation: InstallationRef): Promise<WhmcsListAccountsResponse> {
const links = await this.prisma.whmcsServiceLink.findMany({
where: { installationId: installation.id },
include: {
server: { include: { plan: true } },
},
orderBy: { externalServiceId: 'asc' },
});
const clientLinks = await this.prisma.whmcsClientLink.findMany({
where: { installationId: installation.id },
});
const clientByUserId = new Map(clientLinks.map((link) => [link.userId, link]));
return {
accounts: links.map((link) => ({
externalServiceId: link.externalServiceId,
externalClientId:
clientByUserId.get(link.server.userId)?.externalClientId ?? '',
serverId: link.serverId,
domain: link.server.name,
username: link.server.joinSlug,
status: this.mapLinkStatusToWhmcs(link.status, link.server.status),
planSlug: link.server.plan?.slug ?? 'unknown',
ramMb: link.server.ramMb,
gameCloudStatus: link.server.status,
linkStatus: link.status,
lastReconciledAt: link.lastReconciledAt?.toISOString() ?? null,
})),
};
}
async importService(
installation: InstallationRef,
input: WhmcsImportServiceRequest,
idempotencyKey: string,
): Promise<{
externalServiceId: string;
status: string;
serverId: string;
imported: boolean;
}> {
const existingOp = await this.prisma.integrationOperation.findUnique({
where: { idempotencyKey },
});
if (existingOp?.result) {
return existingOp.result as {
externalServiceId: string;
status: string;
serverId: string;
imported: boolean;
};
}
const existing = await this.prisma.whmcsServiceLink.findUnique({
where: {
installationId_externalServiceId: {
installationId: installation.id,
externalServiceId: input.externalServiceId,
},
},
});
if (existing) {
throw new ConflictException('Service link already exists');
}
const clientLink = await this.prisma.whmcsClientLink.findUnique({
where: {
installationId_externalClientId: {
installationId: installation.id,
externalClientId: input.externalClientId,
},
},
});
if (!clientLink) {
throw new NotFoundException(`Unknown WHMCS client ${input.externalClientId}`);
}
const server = await this.prisma.gameServer.findUnique({
where: { id: input.serverId },
});
if (!server || server.userId !== clientLink.userId) {
throw new NotFoundException('Server not found for client');
}
const existingServerLink = await this.prisma.whmcsServiceLink.findUnique({
where: { serverId: server.id },
});
if (existingServerLink) {
throw new ConflictException('Server is already linked to a WHMCS service');
}
const metadata =
input.expectedPlanSlug !== undefined
? ({ expectedPlanSlug: input.expectedPlanSlug } satisfies Record<string, string>)
: undefined;
const link = await this.prisma.whmcsServiceLink.create({
data: {
installationId: installation.id,
externalServiceId: input.externalServiceId,
externalProductId: input.externalProductId,
serverId: server.id,
status: server.billingSuspendedAt ? 'SUSPENDED' : 'ACTIVE',
metadata: metadata as Prisma.InputJsonValue | undefined,
},
include: { server: { include: { plan: true } } },
});
const result = {
externalServiceId: link.externalServiceId,
status: link.status,
serverId: link.serverId,
imported: true,
};
await this.prisma.integrationOperation.create({
data: {
installationId: installation.id,
idempotencyKey,
operation: 'service:import',
externalRef: input.externalServiceId,
status: 'COMPLETED',
result: result as object,
},
});
return result;
}
async reconcileInstallation(
installation: InstallationRef,
input: WhmcsReconcileRequest,
externalServiceIds?: string[],
): Promise<WhmcsReconcileResponse> {
const runId = randomUUID();
const issues: IssueDraft[] = [];
const repaired: string[] = [];
const links = await this.prisma.whmcsServiceLink.findMany({
where: {
installationId: installation.id,
...(externalServiceIds?.length
? { externalServiceId: { in: externalServiceIds } }
: {}),
},
include: {
server: { include: { plan: true } },
},
});
for (const link of links) {
issues.push(
...this.inspectServiceLink(link, input),
);
}
if (!externalServiceIds?.length) {
issues.push(...(await this.findOrphanServers(installation.id)));
}
if (input.autoRepair && !input.dryRun) {
for (const link of links) {
const repairedCodes = await this.applySafeRepairs(link, input);
repaired.push(...repairedCodes);
}
}
if (!input.dryRun) {
const persistedIssues = await Promise.all(
issues.map((issue) =>
this.prisma.reconciliationIssue.create({
data: {
installationId: installation.id,
runId,
externalServiceId: issue.externalServiceId,
serverId: issue.serverId,
code: issue.code,
severity: issue.severity,
message: issue.message,
details: issue.details as Prisma.InputJsonValue | undefined,
autoRepaired: repaired.some((code) => code === issue.code),
},
}),
),
);
await this.prisma.whmcsServiceLink.updateMany({
where: {
installationId: installation.id,
id: { in: links.map((link) => link.id) },
},
data: { lastReconciledAt: new Date() },
});
const blockingIssues = issues.filter(
(issue) => issue.severity === 'BLOCKING' || issue.severity === 'SECURITY',
);
if (blockingIssues.length > 0) {
await this.events.emitEvent(
installation.id,
'reconciliation_required',
{
runId,
issueCount: issues.length,
blockingCount: blockingIssues.length,
},
`reconciliation:${installation.id}:${runId}`,
);
}
return {
runId,
dryRun: false,
issues: persistedIssues.map((issue) => ({
id: issue.id,
code: issue.code,
severity: issue.severity,
message: issue.message,
externalServiceId: issue.externalServiceId,
serverId: issue.serverId,
details: issue.details as Record<string, unknown> | undefined,
autoRepaired: issue.autoRepaired,
})),
repaired,
};
}
return {
runId,
dryRun: true,
issues: issues.map((issue) => ({
id: randomUUID(),
code: issue.code,
severity: issue.severity,
message: issue.message,
externalServiceId: issue.externalServiceId ?? null,
serverId: issue.serverId ?? null,
details: issue.details,
autoRepaired: false,
})),
repaired: [],
};
}
async reconcileService(
installation: InstallationRef,
externalServiceId: string,
input: WhmcsReconcileRequest,
): Promise<WhmcsReconcileResponse> {
const link = await this.prisma.whmcsServiceLink.findUnique({
where: {
installationId_externalServiceId: {
installationId: installation.id,
externalServiceId,
},
},
});
if (!link) {
throw new NotFoundException(`Unknown WHMCS service ${externalServiceId}`);
}
return this.reconcileInstallation(installation, input, [externalServiceId]);
}
private inspectServiceLink(
link: {
externalServiceId: string;
status: string;
metadata: unknown;
server: {
id: string;
status: string;
ramMb: number;
billingSuspendedAt: Date | null;
joinSlug: string | null;
edition: string;
plan: { slug: string; maxRamMb: number } | null;
};
},
input: WhmcsReconcileRequest,
): IssueDraft[] {
const issues: IssueDraft[] = [];
const server = link.server;
const base = {
externalServiceId: link.externalServiceId,
serverId: server.id,
};
if (link.status === 'SUSPENDED' && !server.billingSuspendedAt) {
issues.push({
...base,
code: 'status_mismatch_suspended',
severity: 'BLOCKING',
message: 'WHMCS link is suspended but server is not billing-suspended',
});
}
if (link.status === 'ACTIVE' && server.billingSuspendedAt) {
issues.push({
...base,
code: 'status_mismatch_active',
severity: 'WARNING',
message: 'WHMCS link is active but server billing suspension is set',
});
}
if (link.status === 'TERMINATED' && server.status !== 'DELETED') {
issues.push({
...base,
code: 'status_mismatch_terminated',
severity: 'DESTRUCTIVE',
message: 'WHMCS link is terminated but GameCloud server still exists',
details: { gameCloudStatus: server.status },
});
}
if (
link.status === 'SUSPENDED' &&
server.status === 'RUNNING' &&
!server.billingSuspendedAt
) {
issues.push({
...base,
code: 'startable_while_suspended',
severity: 'SECURITY',
message: 'Suspended service is still running without billing suspension',
});
}
const metadata = link.metadata as { expectedPlanSlug?: string; expectedRamMb?: number } | null;
const expectedPlanSlug = input.expectedPlanSlug ?? metadata?.expectedPlanSlug;
if (expectedPlanSlug && server.plan && expectedPlanSlug !== server.plan.slug) {
issues.push({
...base,
code: 'plan_mismatch',
severity: 'WARNING',
message: `Expected plan ${expectedPlanSlug} but server has ${server.plan.slug}`,
details: { expectedPlanSlug, actualPlanSlug: server.plan.slug },
});
}
const expectedRamMb = input.expectedRamMb ?? metadata?.expectedRamMb;
if (expectedRamMb && expectedRamMb !== server.ramMb) {
issues.push({
...base,
code: 'ram_mismatch',
severity: 'WARNING',
message: `Expected RAM ${expectedRamMb} MiB but server has ${server.ramMb} MiB`,
details: { expectedRamMb, actualRamMb: server.ramMb },
});
}
if (input.expectedWhmcsStatus) {
const expectedLinkStatus = this.mapWhmcsStatusToLink(input.expectedWhmcsStatus);
if (expectedLinkStatus && link.status !== expectedLinkStatus) {
issues.push({
...base,
code: 'whmcs_status_mismatch',
severity: 'BLOCKING',
message: `WHMCS reports ${input.expectedWhmcsStatus} but link status is ${link.status}`,
details: {
expectedWhmcsStatus: input.expectedWhmcsStatus,
linkStatus: link.status,
},
});
}
}
if (!server.joinSlug) {
issues.push({
...base,
code: 'missing_join_slug',
severity: 'INFO',
message: 'Server has no join slug assigned',
});
} else {
const hostname = formatJoinAddress(server.joinSlug, server.edition as 'JAVA' | 'BEDROCK');
issues.push({
...base,
code: 'join_hostname',
severity: 'INFO',
message: `Join hostname ${hostname}`,
details: { joinHostname: hostname },
});
}
return issues;
}
private async findOrphanServers(installationId: string): Promise<IssueDraft[]> {
const clientUserIds = await this.prisma.whmcsClientLink.findMany({
where: { installationId },
select: { userId: true },
});
const userIds = clientUserIds.map((link) => link.userId);
if (userIds.length === 0) {
return [];
}
const linkedServerIds = await this.prisma.whmcsServiceLink.findMany({
where: { installationId },
select: { serverId: true },
});
const linkedSet = new Set(linkedServerIds.map((link) => link.serverId));
const orphans = await this.prisma.gameServer.findMany({
where: {
userId: { in: userIds },
status: { not: 'DELETED' },
id: { notIn: [...linkedSet] },
},
select: { id: true, name: true, userId: true },
});
return orphans.map((server) => ({
code: 'orphan_server',
severity: 'WARNING' as const,
message: `GameCloud server ${server.name} has no WHMCS service link`,
serverId: server.id,
details: { userId: server.userId },
}));
}
private async applySafeRepairs(
link: {
id: string;
status: string;
serverId: string;
server: {
billingSuspendedAt: Date | null;
status: string;
};
},
input: WhmcsReconcileRequest,
): Promise<string[]> {
const repaired: string[] = [];
if (link.status === 'SUSPENDED' && !link.server.billingSuspendedAt) {
await this.prisma.gameServer.update({
where: { id: link.serverId },
data: {
billingSuspendedAt: new Date(),
billingSuspendReason: 'reconciliation-auto-repair',
},
});
repaired.push('status_mismatch_suspended');
}
if (
link.status === 'ACTIVE' &&
link.server.billingSuspendedAt &&
input.expectedWhmcsStatus === 'Active'
) {
await this.prisma.gameServer.update({
where: { id: link.serverId },
data: {
billingSuspendedAt: null,
billingSuspendReason: null,
},
});
repaired.push('status_mismatch_active');
}
if (
link.status === 'SUSPENDED' &&
link.server.status === 'RUNNING' &&
!link.server.billingSuspendedAt
) {
await this.prisma.gameServer.update({
where: { id: link.serverId },
data: {
billingSuspendedAt: new Date(),
billingSuspendReason: 'reconciliation-security-repair',
},
});
repaired.push('startable_while_suspended');
}
return repaired;
}
private mapLinkStatusToWhmcs(
linkStatus: string,
serverStatus: string,
): 'Active' | 'Suspended' | 'Terminated' | 'Pending' {
if (linkStatus === 'TERMINATED' || serverStatus === 'DELETED') {
return 'Terminated';
}
if (linkStatus === 'SUSPENDED') {
return 'Suspended';
}
if (linkStatus === 'PENDING') {
return 'Pending';
}
return 'Active';
}
private mapWhmcsStatusToLink(
status: string,
): 'PENDING' | 'ACTIVE' | 'SUSPENDED' | 'TERMINATED' | null {
switch (status) {
case 'Pending':
return 'PENDING';
case 'Active':
return 'ACTIVE';
case 'Suspended':
case 'Cancelled':
case 'Fraud':
return 'SUSPENDED';
case 'Terminated':
return 'TERMINATED';
default:
return null;
}
}
}

View File

@@ -0,0 +1,330 @@
import { createHash, randomUUID } from 'node:crypto';
import {
Injectable,
NotFoundException,
} from '@nestjs/common';
import type {
WhmcsUsageAdjustmentRequest,
WhmcsUsageResponse,
} from '@hexahost/contracts';
import type { Prisma } from '@hexahost/database';
import { PrismaService } from '../../prisma/prisma.service';
import { WhmcsEventsService } from './whmcs-events.service';
import { resolveBillingPeriod } from './whmcs-period.util';
type InstallationRef = {
id: string;
};
export type WhmcsUsageMetrics = {
runtime_minutes: number;
memory_gib_minutes: number;
cpu_core_minutes: number;
storage_gib_days: number;
backup_storage_gib_days: number;
egress_gib: number;
extra_port_days: number;
priority_queue_minutes: number;
managed_service_units: number;
};
@Injectable()
export class WhmcsUsageService {
constructor(
private readonly prisma: PrismaService,
private readonly events: WhmcsEventsService,
) {}
async getServiceUsage(
installation: InstallationRef,
externalServiceId: string,
periodStart?: string,
periodEnd?: string,
testMode = false,
): Promise<WhmcsUsageResponse> {
const link = await this.prisma.whmcsServiceLink.findUnique({
where: {
installationId_externalServiceId: {
installationId: installation.id,
externalServiceId,
},
},
include: {
server: { include: { plan: true } },
},
});
if (!link) {
throw new NotFoundException(`Unknown WHMCS service ${externalServiceId}`);
}
const period = resolveBillingPeriod(periodStart, periodEnd);
const existing = await this.prisma.whmcsUsageExport.findUnique({
where: {
installationId_externalServiceId_periodStart_periodEnd_isTestMode: {
installationId: installation.id,
externalServiceId,
periodStart: period.start,
periodEnd: period.end,
isTestMode: testMode,
},
},
});
if (existing) {
return this.toUsageResponse(existing, externalServiceId);
}
const metrics = await this.aggregateMetrics(
link.serverId,
period.start,
period.end,
link.server.plan?.maxCpuCores ?? 1,
link.server.plan?.maxStorageMb ?? 5120,
link.server.plan?.queuePriority ?? 0,
);
const contentHash = this.hashMetrics(metrics);
const usageUpdate = this.toUsageUpdate(metrics);
const created = await this.prisma.whmcsUsageExport.create({
data: {
installationId: installation.id,
externalServiceId,
serverId: link.serverId,
periodStart: period.start,
periodEnd: period.end,
metrics: metrics as Prisma.InputJsonValue,
contentHash,
status: testMode ? 'DRAFT' : 'EXPORTED',
isTestMode: testMode,
exportedAt: testMode ? null : new Date(),
},
});
if (!testMode) {
await this.events.emitEvent(
installation.id,
'usage_period_closed',
{
exportId: created.id,
externalServiceId,
periodStart: period.start.toISOString(),
periodEnd: period.end.toISOString(),
contentHash,
},
`usage:${installation.id}:${externalServiceId}:${period.start.toISOString()}:${period.end.toISOString()}`,
);
}
return this.toUsageResponse(created, externalServiceId, usageUpdate);
}
async createAdjustment(
installation: InstallationRef,
externalServiceId: string,
input: WhmcsUsageAdjustmentRequest,
idempotencyKey: string,
): Promise<WhmcsUsageResponse> {
const existingOp = await this.prisma.integrationOperation.findUnique({
where: { idempotencyKey },
});
if (existingOp?.result) {
return existingOp.result as WhmcsUsageResponse;
}
const link = await this.prisma.whmcsServiceLink.findUnique({
where: {
installationId_externalServiceId: {
installationId: installation.id,
externalServiceId,
},
},
});
if (!link) {
throw new NotFoundException(`Unknown WHMCS service ${externalServiceId}`);
}
const periodStart = new Date(input.periodStart);
const periodEnd = new Date(input.periodEnd);
const baseExport = await this.prisma.whmcsUsageExport.findUnique({
where: {
installationId_externalServiceId_periodStart_periodEnd_isTestMode: {
installationId: installation.id,
externalServiceId,
periodStart,
periodEnd,
isTestMode: input.testMode,
},
},
});
if (!baseExport) {
throw new NotFoundException('No usage export exists for this period');
}
const baseMetrics = baseExport.metrics as WhmcsUsageMetrics;
const adjustedMetrics = { ...baseMetrics };
for (const adjustment of input.adjustments) {
const key = adjustment.metricKey as keyof WhmcsUsageMetrics;
if (key in adjustedMetrics) {
adjustedMetrics[key] = roundMetric(adjustedMetrics[key] + adjustment.delta);
}
}
const contentHash = this.hashMetrics(adjustedMetrics);
const usageUpdate = this.toUsageUpdate(adjustedMetrics);
const adjustmentExport = await this.prisma.whmcsUsageExport.update({
where: { id: baseExport.id },
data: {
metrics: adjustedMetrics as Prisma.InputJsonValue,
contentHash,
status: 'ADJUSTED',
adjustmentOfId: baseExport.adjustmentOfId ?? baseExport.id,
exportedAt: new Date(),
},
});
const response = this.toUsageResponse(
adjustmentExport,
externalServiceId,
usageUpdate,
input.adjustments,
);
await this.prisma.integrationOperation.create({
data: {
installationId: installation.id,
idempotencyKey,
operation: 'usage:adjustment',
externalRef: externalServiceId,
status: 'COMPLETED',
result: response as object,
},
});
return response;
}
private async aggregateMetrics(
serverId: string,
periodStart: Date,
periodEnd: Date,
cpuCores: number,
storageMb: number,
queuePriority: number,
): Promise<WhmcsUsageMetrics> {
const usageRecords = await this.prisma.usageRecord.findMany({
where: {
serverId,
periodStart: { gte: periodStart },
periodEnd: { lte: periodEnd },
},
});
let runtimeSeconds = 0;
let memoryMbSeconds = 0;
for (const record of usageRecords) {
runtimeSeconds += record.durationSeconds;
memoryMbSeconds += record.ramMb * record.durationSeconds;
}
const periodDays = Math.max(
(periodEnd.getTime() - periodStart.getTime()) / (24 * 60 * 60 * 1000),
1,
);
const backups = await this.prisma.serverBackup.findMany({
where: {
serverId,
status: 'AVAILABLE',
completedAt: {
gte: periodStart,
lt: periodEnd,
},
},
select: { sizeBytes: true },
});
let backupStorageGiBDays = 0;
for (const backup of backups) {
const sizeGiB = Number(backup.sizeBytes ?? 0n) / (1024 ** 3);
backupStorageGiBDays += sizeGiB * periodDays;
}
const runtimeMinutes = roundMetric(runtimeSeconds / 60);
const memoryGibMinutes = roundMetric(memoryMbSeconds / 1024 / 60);
const cpuCoreMinutes = roundMetric((runtimeSeconds / 60) * cpuCores);
const storageGibDays = roundMetric((storageMb / 1024) * periodDays);
const priorityQueueMinutes =
queuePriority > 0 ? roundMetric(runtimeMinutes * (queuePriority / 10)) : 0;
return {
runtime_minutes: runtimeMinutes,
memory_gib_minutes: memoryGibMinutes,
cpu_core_minutes: cpuCoreMinutes,
storage_gib_days: storageGibDays,
backup_storage_gib_days: roundMetric(backupStorageGiBDays),
egress_gib: 0,
extra_port_days: 0,
priority_queue_minutes: priorityQueueMinutes,
managed_service_units: 0,
};
}
private hashMetrics(metrics: WhmcsUsageMetrics): string {
return createHash('sha256').update(JSON.stringify(metrics)).digest('hex');
}
private toUsageUpdate(metrics: WhmcsUsageMetrics) {
return {
diskUsageGiB: metrics.storage_gib_days,
bandwidthGiB: metrics.egress_gib,
};
}
private toUsageResponse(
exportRow: {
id: string;
serverId: string;
periodStart: Date;
periodEnd: Date;
metrics: unknown;
contentHash: string;
status: 'DRAFT' | 'EXPORTED' | 'ADJUSTED';
isTestMode: boolean;
},
externalServiceId: string,
usageUpdate?: { diskUsageGiB: number; bandwidthGiB: number },
adjustments: WhmcsUsageResponse['adjustments'] = [],
): WhmcsUsageResponse {
const metrics = exportRow.metrics as WhmcsUsageMetrics;
return {
exportId: exportRow.id,
externalServiceId,
serverId: exportRow.serverId,
periodStart: exportRow.periodStart.toISOString(),
periodEnd: exportRow.periodEnd.toISOString(),
metrics,
usageUpdate: usageUpdate ?? this.toUsageUpdate(metrics),
contentHash: exportRow.contentHash,
status: exportRow.status,
isTestMode: exportRow.isTestMode,
adjustments,
};
}
}
function roundMetric(value: number): number {
return Math.round(value * 1000) / 1000;
}

View File

@@ -0,0 +1,46 @@
import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
describe('whmcs phase e/f contract', () => {
it('documents reconciliation, events and usage routes', () => {
const whmcsSource = readFileSync(
join(process.cwd(), 'src', 'integrations/whmcs/whmcs-integration.controller.ts'),
'utf8',
);
assert.ok(whmcsSource.includes("'accounts'"));
assert.ok(whmcsSource.includes("'services/import'"));
assert.ok(whmcsSource.includes("'reconcile'"));
assert.ok(whmcsSource.includes("'services/:externalServiceId/reconcile'"));
assert.ok(whmcsSource.includes("'events'"));
assert.ok(whmcsSource.includes("'events/ack'"));
assert.ok(whmcsSource.includes("'services/:externalServiceId/usage'"));
assert.ok(whmcsSource.includes("'services/:externalServiceId/usage/adjustments'"));
const reconcileSource = readFileSync(
join(process.cwd(), 'src', 'integrations/whmcs/whmcs-reconciliation.service.ts'),
'utf8',
);
assert.ok(reconcileSource.includes('listAccounts'));
assert.ok(reconcileSource.includes('importService'));
assert.ok(reconcileSource.includes('reconcileInstallation'));
const usageSource = readFileSync(
join(process.cwd(), 'src', 'integrations/whmcs/whmcs-usage.service.ts'),
'utf8',
);
assert.ok(usageSource.includes('runtime_minutes'));
assert.ok(usageSource.includes('createAdjustment'));
});
it('documents WHMCS PHP ListAccounts and usage hooks', () => {
const phpSource = readFileSync(
join(process.cwd(), '..', '..', 'integrations/whmcs/modules/servers/hexagamecloud/hexagamecloud.php'),
'utf8',
);
assert.ok(phpSource.includes('hexagamecloud_ListAccounts'));
assert.ok(phpSource.includes('hexagamecloud_MetricProvider'));
assert.ok(phpSource.includes('hexagamecloud_UsageUpdate'));
});
});

View File

@@ -0,0 +1,190 @@
import { randomUUID } from 'node:crypto';
import { prisma } from '@hexahost/database';
import { logger } from '../logger';
function previousBillingPeriod(reference = new Date()): { start: Date; end: Date } {
const start = new Date(
Date.UTC(reference.getUTCFullYear(), reference.getUTCMonth() - 1, 1),
);
const end = new Date(Date.UTC(reference.getUTCFullYear(), reference.getUTCMonth(), 1));
return { start, end };
}
export async function processWhmcsReconciliationTick(): Promise<void> {
const now = new Date();
if (now.getUTCHours() !== 3) {
return;
}
const installations = await prisma.whmcsInstallation.findMany({
where: { isActive: true },
select: { id: true, integrationId: true },
});
for (const installation of installations) {
try {
await runInstallationReconciliation(installation.id, installation.integrationId);
} catch (error) {
logger.error(
{ err: error, installationId: installation.id },
'WHMCS reconciliation tick failed',
);
}
}
}
async function runInstallationReconciliation(
installationId: string,
integrationId: string,
): Promise<void> {
const runId = randomUUID();
const links = await prisma.whmcsServiceLink.findMany({
where: { installationId },
include: { server: { include: { plan: true } } },
});
const issues: Array<{
code: string;
severity: 'INFO' | 'WARNING' | 'BLOCKING' | 'DESTRUCTIVE' | 'SECURITY';
message: string;
externalServiceId?: string;
serverId?: string;
}> = [];
for (const link of links) {
if (link.status === 'SUSPENDED' && !link.server.billingSuspendedAt) {
issues.push({
code: 'status_mismatch_suspended',
severity: 'BLOCKING',
message: 'Suspended link without billing suspension',
externalServiceId: link.externalServiceId,
serverId: link.serverId,
});
}
if (link.status === 'TERMINATED' && link.server.status !== 'DELETED') {
issues.push({
code: 'status_mismatch_terminated',
severity: 'DESTRUCTIVE',
message: 'Terminated link with active server',
externalServiceId: link.externalServiceId,
serverId: link.serverId,
});
}
}
if (issues.length === 0) {
await prisma.whmcsServiceLink.updateMany({
where: { installationId },
data: { lastReconciledAt: new Date() },
});
return;
}
await prisma.reconciliationIssue.createMany({
data: issues.map((issue) => ({
installationId,
runId,
externalServiceId: issue.externalServiceId,
serverId: issue.serverId,
code: issue.code,
severity: issue.severity,
message: issue.message,
})),
});
const blockingCount = issues.filter(
(issue) => issue.severity === 'BLOCKING' || issue.severity === 'SECURITY',
).length;
if (blockingCount > 0) {
try {
await prisma.integrationEvent.create({
data: {
installationId,
eventType: 'reconciliation_required',
payload: {
runId,
integrationId,
issueCount: issues.length,
blockingCount,
},
dedupeKey: `reconciliation:${installationId}:${runId}`,
},
});
} catch {
// duplicate dedupe
}
}
await prisma.whmcsServiceLink.updateMany({
where: { installationId },
data: { lastReconciledAt: new Date() },
});
logger.info(
{ installationId, runId, issueCount: issues.length },
'WHMCS reconciliation tick completed',
);
}
export async function processWhmcsUsagePeriodTick(): Promise<void> {
const now = new Date();
if (now.getUTCDate() !== 1 || now.getUTCHours() > 1) {
return;
}
const period = previousBillingPeriod(now);
const installations = await prisma.whmcsInstallation.findMany({
where: { isActive: true },
select: { id: true },
});
for (const installation of installations) {
const links = await prisma.whmcsServiceLink.findMany({
where: {
installationId: installation.id,
status: { in: ['ACTIVE', 'SUSPENDED'] },
},
select: { externalServiceId: true, serverId: true },
});
for (const link of links) {
const existing = await prisma.whmcsUsageExport.findUnique({
where: {
installationId_externalServiceId_periodStart_periodEnd_isTestMode: {
installationId: installation.id,
externalServiceId: link.externalServiceId,
periodStart: period.start,
periodEnd: period.end,
isTestMode: false,
},
},
});
if (existing) {
continue;
}
try {
await prisma.integrationEvent.create({
data: {
installationId: installation.id,
eventType: 'usage_period_ready',
payload: {
externalServiceId: link.externalServiceId,
serverId: link.serverId,
periodStart: period.start.toISOString(),
periodEnd: period.end.toISOString(),
},
dedupeKey: `usage-ready:${installation.id}:${link.externalServiceId}:${period.start.toISOString()}`,
},
});
} catch {
// duplicate
}
}
}
}

View File

@@ -12,6 +12,7 @@ import { processBackupJob, processScheduledBackupsTick } from './handlers/server
import { processLifecycleJob } from './handlers/server-lifecycle';
import { processProvisionServerJob } from './handlers/server-provisioning';
import { processStartQueueTick } from './handlers/start-queue';
import { processWhmcsReconciliationTick, processWhmcsUsagePeriodTick } from './handlers/whmcs-sync';
import { startHealthServer } from './health';
import { logger } from './logger';
import { closeRedisConnections } from './redis';
@@ -110,6 +111,12 @@ async function bootstrap(): Promise<void> {
void processDnsSyncTick().catch((error: Error) => {
logger.error({ err: error }, 'DNS sync tick failed');
});
void processWhmcsReconciliationTick().catch((error: Error) => {
logger.error({ err: error }, 'WHMCS reconciliation tick failed');
});
void processWhmcsUsagePeriodTick().catch((error: Error) => {
logger.error({ err: error }, 'WHMCS usage period tick failed');
});
}, 60_000);
const shutdown = async (signal: string): Promise<void> => {

File diff suppressed because one or more lines are too long