diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 471c949..7959bb8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -210,6 +210,9 @@ importers: passport-local: specifier: ^1.0.0 version: 1.0.0 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 reflect-metadata: specifier: ^0.2.2 version: 0.2.2 @@ -1892,6 +1895,11 @@ packages: transitivePeerDependencies: - encoding + /@opentelemetry/api@1.9.1: + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + dev: false + /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -3829,6 +3837,10 @@ packages: engines: {node: '>=8'} dev: true + /bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + dev: false + /bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} dependencies: @@ -7127,6 +7139,14 @@ packages: optionalDependencies: fsevents: 2.3.3 + /prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + dependencies: + '@opentelemetry/api': 1.9.1 + tdigest: 0.1.2 + dev: false + /prompts@2.4.2: resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} engines: {node: '>= 6'} @@ -8009,6 +8029,12 @@ packages: yallist: 4.0.0 dev: false + /tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + dependencies: + bintrees: 1.0.2 + dev: false + /terser-webpack-plugin@5.6.1(webpack@5.97.1): resolution: {integrity: sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==} engines: {node: '>= 10.13.0'} diff --git a/server/package.json b/server/package.json index d220ac7..2e210dc 100644 --- a/server/package.json +++ b/server/package.json @@ -30,10 +30,10 @@ "@nestjs/jwt": "^10.2.0", "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.3.8", + "@nestjs/platform-socket.io": "^10.3.8", "@nestjs/schedule": "^4.0.2", "@nestjs/swagger": "^7.3.1", "@nestjs/websockets": "^10.3.8", - "@nestjs/platform-socket.io": "^10.3.8", "@prisma/client": "^5.14.0", "@socket.io/redis-adapter": "^8.3.0", "bcrypt": "^5.1.1", @@ -44,6 +44,7 @@ "passport": "^0.7.0", "passport-jwt": "^4.0.1", "passport-local": "^1.0.0", + "prom-client": "^15.1.3", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "socket.io": "^4.7.5", @@ -73,13 +74,19 @@ "typescript": "^5.4.5" }, "jest": { - "moduleFileExtensions": ["js", "json", "ts"], + "moduleFileExtensions": [ + "js", + "json", + "ts" + ], "rootDir": "src", "testRegex": ".*\\.spec\\.ts$", "transform": { "^.+\\.(t|j)s$": "ts-jest" }, - "collectCoverageFrom": ["**/*.(t|j)s"], + "collectCoverageFrom": [ + "**/*.(t|j)s" + ], "coverageDirectory": "../coverage", "testEnvironment": "node", "moduleNameMapper": { diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 323affb..57e4a7b 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -14,6 +14,7 @@ import { InvitationsModule } from './modules/invitations/invitations.module'; import { NotificationsModule } from './modules/notifications/notifications.module'; import { LeaderboardsModule } from './modules/leaderboards/leaderboards.module'; import { WebSocketGatewayModule } from './websocket/websocket.module'; +import { MetricsModule } from './metrics/metrics.module'; import appConfig from './config/app.config'; import databaseConfig from './config/database.config'; import jwtConfig from './config/jwt.config'; @@ -29,6 +30,7 @@ import redisConfig from './config/redis.config'; ScheduleModule.forRoot(), DatabaseModule, WebSocketGatewayModule, + MetricsModule, AuthModule, UsersModule, GamesModule, diff --git a/server/src/main.ts b/server/src/main.ts index 65eab97..2f8919d 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -13,7 +13,7 @@ async function bootstrap() { const port = configService.get('PORT', 3001); const clientOrigin = configService.get('CLIENT_ORIGIN', 'http://localhost:5173'); - app.setGlobalPrefix('api'); + app.setGlobalPrefix('api', { exclude: ['metrics'] }); // /metrics is cluster-internal (not proxied by nginx) app.enableCors({ origin: clientOrigin, diff --git a/server/src/metrics/metrics.module.ts b/server/src/metrics/metrics.module.ts new file mode 100644 index 0000000..572c8a7 --- /dev/null +++ b/server/src/metrics/metrics.module.ts @@ -0,0 +1,37 @@ +import { Controller, Get, Header, Module } from '@nestjs/common'; +import { Interval } from '@nestjs/schedule'; +import { Injectable } from '@nestjs/common'; +import { DatabaseModule } from '../database/database.module'; +import { PrismaService } from '../database/prisma.service'; +import { gamesActive, registry } from './metrics'; + +@Controller('metrics') +export class MetricsController { + @Get() + @Header('Content-Type', 'text/plain; version=0.0.4; charset=utf-8') + async metrics(): Promise { + return registry.metrics(); + } +} + +@Injectable() +export class MetricsSampler { + constructor(private readonly prisma: PrismaService) {} + + /** One tiny indexed count every 30s keeps the gauge honest across restarts. */ + @Interval(30_000) + async sampleActiveGames(): Promise { + try { + gamesActive.set(await this.prisma.game.count({ where: { status: 'active' } })); + } catch { + // Sampling must never crash the app; the next tick retries. + } + } +} + +@Module({ + imports: [DatabaseModule], + controllers: [MetricsController], + providers: [MetricsSampler], +}) +export class MetricsModule {} diff --git a/server/src/metrics/metrics.ts b/server/src/metrics/metrics.ts new file mode 100644 index 0000000..9f7a0f0 --- /dev/null +++ b/server/src/metrics/metrics.ts @@ -0,0 +1,34 @@ +/** + * Prometheus metrics, hand-picked for a 1 vCPU host: what is being used + * (games, sockets) and what can degrade it (Stockfish work). No default + * process metrics: the scrape must stay tiny. + */ +import { Gauge, Histogram, Registry } from 'prom-client'; + +export const registry = new Registry(); + +export const gamesActive = new Gauge({ + name: 'chesskernel_games_active', + help: 'Games currently in active status', + registers: [registry], +}); + +export const socketConnections = new Gauge({ + name: 'chesskernel_socket_connections', + help: 'Socket.IO clients currently connected', + registers: [registry], +}); + +export const stockfishRunning = new Gauge({ + name: 'chesskernel_stockfish_running', + help: 'Stockfish evaluations currently in flight', + registers: [registry], +}); + +export const stockfishDuration = new Histogram({ + name: 'chesskernel_stockfish_duration_seconds', + help: 'Stockfish call duration by operation', + labelNames: ['op'], + buckets: [0.25, 1, 3, 10, 30], + registers: [registry], +}); diff --git a/server/src/modules/analysis/stockfish.service.ts b/server/src/modules/analysis/stockfish.service.ts index d2d6ecd..024c3a6 100644 --- a/server/src/modules/analysis/stockfish.service.ts +++ b/server/src/modules/analysis/stockfish.service.ts @@ -1,5 +1,6 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { spawn, ChildProcess } from 'child_process'; +import { stockfishDuration, stockfishRunning } from '../../metrics/metrics'; import { join } from 'path'; import { existsSync } from 'fs'; @@ -39,7 +40,7 @@ export class StockfishService implements OnModuleDestroy { } async evaluatePosition(fen: string, depth = 18): Promise { - return new Promise((resolve, reject) => { + return this.timed('evaluate', () => new Promise((resolve, reject) => { let process: ChildProcess; try { process = spawn(this.findStockfish(), [], { stdio: 'pipe' }); @@ -103,7 +104,19 @@ export class StockfishService implements OnModuleDestroy { ].join('\n') + '\n'; process.stdin?.write(commands); - }); + })); + } + + /** Tracks in-flight count and duration of every engine call. */ + private async timed(op: string, run: () => Promise): Promise { + stockfishRunning.inc(); + const stop = stockfishDuration.startTimer({ op }); + try { + return await run(); + } finally { + stop(); + stockfishRunning.dec(); + } } async getBestMove( @@ -112,7 +125,7 @@ export class StockfishService implements OnModuleDestroy { moveTimeMs: number, depth?: number, ): Promise { - return new Promise((resolve, reject) => { + return this.timed('bestmove', () => new Promise((resolve, reject) => { let process: ChildProcess; try { process = spawn(this.findStockfish(), [], { stdio: 'pipe' }); @@ -165,6 +178,6 @@ export class StockfishService implements OnModuleDestroy { ].join('\n') + '\n'; process.stdin?.write(commands); - }); + })); } } diff --git a/server/src/websocket/game.gateway.ts b/server/src/websocket/game.gateway.ts index 940fa3b..dadfa04 100644 --- a/server/src/websocket/game.gateway.ts +++ b/server/src/websocket/game.gateway.ts @@ -7,6 +7,7 @@ import { ConnectedSocket, MessageBody, } from '@nestjs/websockets'; +import { socketConnections } from '../metrics/metrics'; import { UseGuards, Logger } from '@nestjs/common'; import { Server, Socket } from 'socket.io'; import { JwtService } from '@nestjs/jwt'; @@ -45,6 +46,7 @@ export class GameGateway implements OnGatewayConnection, OnGatewayDisconnect { ) {} async handleConnection(socket: Socket) { + socketConnections.inc(); const token = socket.handshake.auth?.token as string | undefined; if (!token) { socket.disconnect(true); @@ -65,6 +67,7 @@ export class GameGateway implements OnGatewayConnection, OnGatewayDisconnect { } async handleDisconnect(socket: Socket) { + socketConnections.dec(); const userId = socket.data.userId as string | undefined; if (!userId) return;