diff --git a/.gitignore b/.gitignore index 5d417368a..feb2c4f8d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ coverage .turbo .databricks + +.superset/config.json diff --git a/apps/dev-playground/app.yaml b/apps/dev-playground/app.yaml index e58e71a31..7b57e4ff8 100644 --- a/apps/dev-playground/app.yaml +++ b/apps/dev-playground/app.yaml @@ -5,6 +5,8 @@ env: valueFrom: genie-space - name: DATABRICKS_SERVING_ENDPOINT_NAME valueFrom: serving-endpoint + - name: DATABRICKS_JOB_ID + valueFrom: job # Files plugin manifest declares a static DATABRICKS_VOLUME_FILES # requirement; keep it bound so appkit's runtime validation passes # even though the policy harness below uses its own keys. diff --git a/apps/dev-playground/appkit.plugins.json b/apps/dev-playground/appkit.plugins.json new file mode 100644 index 000000000..5d7b902a0 --- /dev/null +++ b/apps/dev-playground/appkit.plugins.json @@ -0,0 +1,92 @@ +{ + "$schema": "https://databricks.github.io/appkit/schemas/template-plugins.schema.json", + "version": "1.0", + "plugins": { + "analytics": { + "name": "analytics", + "displayName": "Analytics Plugin", + "description": "SQL query execution against Databricks SQL Warehouses", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "sql_warehouse", + "alias": "SQL Warehouse", + "resourceKey": "sql-warehouse", + "description": "SQL Warehouse for executing analytics queries", + "permission": "CAN_USE", + "fields": { + "id": { + "env": "DATABRICKS_WAREHOUSE_ID", + "description": "SQL Warehouse ID" + } + } + } + ], + "optional": [] + }, + "requiredByTemplate": true + }, + "files": { + "name": "files", + "displayName": "Files Plugin", + "description": "File operations against Databricks Volumes and Unity Catalog", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "volume", + "alias": "Files", + "resourceKey": "files", + "description": "Permission to write to volumes", + "permission": "WRITE_VOLUME", + "fields": { + "path": { + "env": "DATABRICKS_VOLUME_FILES", + "description": "Volume path for file storage (e.g. /Volumes/catalog/schema/volume_name)" + } + } + } + ], + "optional": [] + }, + "requiredByTemplate": true + }, + "genie": { + "name": "genie", + "displayName": "Genie Plugin", + "description": "AI/BI Genie space integration for natural language data queries", + "package": "@databricks/appkit", + "resources": { + "required": [ + { + "type": "genie_space", + "alias": "Genie Space", + "resourceKey": "genie-space", + "description": "Genie Space for AI-powered data queries. Space IDs configured via plugin config.", + "permission": "CAN_RUN", + "fields": { + "id": { + "env": "DATABRICKS_GENIE_SPACE_ID", + "description": "Default Genie Space ID" + } + } + } + ], + "optional": [] + }, + "requiredByTemplate": true + }, + "server": { + "name": "server", + "displayName": "Server Plugin", + "description": "HTTP server with Express, static file serving, and Vite dev mode support", + "package": "@databricks/appkit", + "resources": { + "required": [], + "optional": [] + }, + "requiredByTemplate": true + } + } +} diff --git a/apps/dev-playground/client/src/routeTree.gen.ts b/apps/dev-playground/client/src/routeTree.gen.ts index 45e280700..479e4e58e 100644 --- a/apps/dev-playground/client/src/routeTree.gen.ts +++ b/apps/dev-playground/client/src/routeTree.gen.ts @@ -18,6 +18,7 @@ import { Route as ReconnectRouteRouteImport } from './routes/reconnect.route' import { Route as PolicyMatrixRouteRouteImport } from './routes/policy-matrix.route' import { Route as LakebaseRouteRouteImport } from './routes/lakebase.route' import { Route as JobsRouteRouteImport } from './routes/jobs.route' +import { Route as InternalTelemetryRouteRouteImport } from './routes/internal-telemetry.route' import { Route as GenieRouteRouteImport } from './routes/genie.route' import { Route as FilesRouteRouteImport } from './routes/files.route' import { Route as DataVisualizationRouteRouteImport } from './routes/data-visualization.route' @@ -71,6 +72,11 @@ const JobsRouteRoute = JobsRouteRouteImport.update({ path: '/jobs', getParentRoute: () => rootRouteImport, } as any) +const InternalTelemetryRouteRoute = InternalTelemetryRouteRouteImport.update({ + id: '/internal-telemetry', + path: '/internal-telemetry', + getParentRoute: () => rootRouteImport, +} as any) const GenieRouteRoute = GenieRouteRouteImport.update({ id: '/genie', path: '/genie', @@ -115,6 +121,7 @@ export interface FileRoutesByFullPath { '/data-visualization': typeof DataVisualizationRouteRoute '/files': typeof FilesRouteRoute '/genie': typeof GenieRouteRoute + '/internal-telemetry': typeof InternalTelemetryRouteRoute '/jobs': typeof JobsRouteRoute '/lakebase': typeof LakebaseRouteRoute '/policy-matrix': typeof PolicyMatrixRouteRoute @@ -133,6 +140,7 @@ export interface FileRoutesByTo { '/data-visualization': typeof DataVisualizationRouteRoute '/files': typeof FilesRouteRoute '/genie': typeof GenieRouteRoute + '/internal-telemetry': typeof InternalTelemetryRouteRoute '/jobs': typeof JobsRouteRoute '/lakebase': typeof LakebaseRouteRoute '/policy-matrix': typeof PolicyMatrixRouteRoute @@ -152,6 +160,7 @@ export interface FileRoutesById { '/data-visualization': typeof DataVisualizationRouteRoute '/files': typeof FilesRouteRoute '/genie': typeof GenieRouteRoute + '/internal-telemetry': typeof InternalTelemetryRouteRoute '/jobs': typeof JobsRouteRoute '/lakebase': typeof LakebaseRouteRoute '/policy-matrix': typeof PolicyMatrixRouteRoute @@ -172,6 +181,7 @@ export interface FileRouteTypes { | '/data-visualization' | '/files' | '/genie' + | '/internal-telemetry' | '/jobs' | '/lakebase' | '/policy-matrix' @@ -190,6 +200,7 @@ export interface FileRouteTypes { | '/data-visualization' | '/files' | '/genie' + | '/internal-telemetry' | '/jobs' | '/lakebase' | '/policy-matrix' @@ -208,6 +219,7 @@ export interface FileRouteTypes { | '/data-visualization' | '/files' | '/genie' + | '/internal-telemetry' | '/jobs' | '/lakebase' | '/policy-matrix' @@ -227,6 +239,7 @@ export interface RootRouteChildren { DataVisualizationRouteRoute: typeof DataVisualizationRouteRoute FilesRouteRoute: typeof FilesRouteRoute GenieRouteRoute: typeof GenieRouteRoute + InternalTelemetryRouteRoute: typeof InternalTelemetryRouteRoute JobsRouteRoute: typeof JobsRouteRoute LakebaseRouteRoute: typeof LakebaseRouteRoute PolicyMatrixRouteRoute: typeof PolicyMatrixRouteRoute @@ -303,6 +316,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof JobsRouteRouteImport parentRoute: typeof rootRouteImport } + '/internal-telemetry': { + id: '/internal-telemetry' + path: '/internal-telemetry' + fullPath: '/internal-telemetry' + preLoaderRoute: typeof InternalTelemetryRouteRouteImport + parentRoute: typeof rootRouteImport + } '/genie': { id: '/genie' path: '/genie' @@ -363,6 +383,7 @@ const rootRouteChildren: RootRouteChildren = { DataVisualizationRouteRoute: DataVisualizationRouteRoute, FilesRouteRoute: FilesRouteRoute, GenieRouteRoute: GenieRouteRoute, + InternalTelemetryRouteRoute: InternalTelemetryRouteRoute, JobsRouteRoute: JobsRouteRoute, LakebaseRouteRoute: LakebaseRouteRoute, PolicyMatrixRouteRoute: PolicyMatrixRouteRoute, diff --git a/apps/dev-playground/client/src/routes/__root.tsx b/apps/dev-playground/client/src/routes/__root.tsx index db42fdafb..98f440d2a 100644 --- a/apps/dev-playground/client/src/routes/__root.tsx +++ b/apps/dev-playground/client/src/routes/__root.tsx @@ -72,6 +72,14 @@ function RootComponent() { Telemetry + + + + + +

diff --git a/apps/dev-playground/client/src/routes/internal-telemetry.route.tsx b/apps/dev-playground/client/src/routes/internal-telemetry.route.tsx new file mode 100644 index 000000000..f627c58db --- /dev/null +++ b/apps/dev-playground/client/src/routes/internal-telemetry.route.tsx @@ -0,0 +1,355 @@ +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, + Button, + Card, +} from "@databricks/appkit-ui/react"; +import { createFileRoute } from "@tanstack/react-router"; +import { Check, Copy, Loader2 } from "lucide-react"; +import { useState } from "react"; + +export const Route = createFileRoute("/internal-telemetry")({ + component: InternalTelemetryRoute, +}); + +const BASE = "/api/internal-telemetry-debug"; + +const ACTIONS = [ + { + id: "startup", + title: "Send APP_STARTUP", + description: + "Sends an AppkitLog with event_name=APP_STARTUP. Mirrors what createApp emits at boot.", + endpoint: `${BASE}/startup`, + }, + { + id: "heartbeat", + title: "Send HEARTBEAT", + description: + "Sends an AppkitLog with event_name=HEARTBEAT. Bypasses the periodic heartbeat timer.", + endpoint: `${BASE}/heartbeat`, + }, + { + id: "record", + title: "Record sample request metrics", + description: + "Adds a synthetic request to the in-memory aggregator. Run a few times before flushing.", + endpoint: `${BASE}/request-metrics-record`, + body: { + method: "GET", + endpoint: "/api/sample/:id", + statusCode: 200, + latencyMs: 42, + }, + }, + { + id: "flush", + title: "Flush REQUEST_METRICS", + description: + "Drains the request metrics aggregator and sends one AppkitLog per endpoint.", + endpoint: `${BASE}/request-metrics-flush`, + }, +] as const; + +type DispatchRequest = { + url: string; + method: string; + headers: Record; + body: string; +}; + +type DispatchResponse = { + status: number; + statusText: string; + body: string; +}; + +type ActionResult = { + ok?: boolean; + error?: string; + action?: string; + message?: string; + request?: DispatchRequest; + response?: DispatchResponse; + curl?: string; + recorded?: unknown; +}; + +function InternalTelemetryRoute() { + const [loading, setLoading] = useState(null); + const [results, setResults] = useState>({}); + + const run = async ( + id: string, + endpoint: string, + body?: Record, + ) => { + setLoading(id); + try { + const response = await fetch(endpoint, { + method: "POST", + headers: body ? { "Content-Type": "application/json" } : {}, + body: body ? JSON.stringify(body) : undefined, + }); + const data = (await response.json()) as ActionResult; + setResults((prev) => ({ ...prev, [id]: data })); + } catch (error) { + setResults((prev) => ({ + ...prev, + [id]: { error: error instanceof Error ? error.message : String(error) }, + })); + } finally { + setLoading(null); + } + }; + + return ( +
+
+
+

Internal Telemetry

+

+ Manually trigger AppKit's internal telemetry events to verify the + pipeline end-to-end. Each action shows the exact request that was + POSTed to the workspace's /telemetry endpoint, the + response, and a curl command you can run locally. +

+
+ +
+ {ACTIONS.map((action) => { + const result = results[action.id]; + const isLoading = loading === action.id; + return ( + +
+
+

+ {action.title} +

+

+ {action.description} +

+
+ +
+ + {result && } + + ); + })} +
+
+
+ ); +} + +function ResultDetails({ result }: { result: ActionResult }) { + const status = statusBadge(result); + const items: Array<{ + value: string; + title: string; + content: React.ReactNode; + }> = []; + + if (result.recorded !== undefined) { + items.push({ + value: "recorded", + title: "Recorded", + content: {stringify(result.recorded)}, + }); + } + if (result.request) { + items.push({ + value: "request", + title: "Request", + content: ( +
+
+ {result.request.method} {result.request.url} +
+ + {stringify(result.request.headers)} + + + {prettyJson(result.request.body)} + +
+ ), + }); + } + if (result.response) { + items.push({ + value: "response", + title: "Response", + content: ( +
+
+ HTTP {result.response.status} {result.response.statusText} +
+ + {result.response.body || "(empty)"} + +
+ ), + }); + } + + return ( +
+
+ {status.label} +
+ {result.message && ( +
{result.message}
+ )} + {result.error && ( + 
+          {result.error}
+
+ )} + {items.length > 0 && ( + + {items.map((item) => ( + + + {item.title} + + {item.content} + + ))} + + )} + {result.curl && } +
+ ); +} + +function statusBadge(result: ActionResult): { + label: string; + className: string; +} { + if (result.error) { + return { + label: `Error: ${result.error}`, + className: "bg-red-50 text-red-800 border-red-200", + }; + } + if (result.response) { + const code = result.response.status; + const ok = code >= 200 && code < 300; + return { + label: `${ok ? "Success" : "Failed"} — HTTP ${code} ${result.response.statusText}`, + className: ok + ? "bg-green-50 text-green-800 border-green-200" + : "bg-yellow-50 text-yellow-900 border-yellow-200", + }; + } + if (result.ok) { + return { + label: result.message ?? "Done", + className: "bg-green-50 text-green-800 border-green-200", + }; + } + return { + label: "Done", + className: "bg-gray-50 text-gray-800 border-gray-200", + }; +} + +function Subsection({ + title, + children, +}: { + title: string; + children: React.ReactNode; +}) { + return ( +
+
+ {title} +
+ {children} +
+ ); +} + +function CodeBlock({ children }: { children: React.ReactNode }) { + return ( + 
+      {children}
+
+ ); +} + +function CurlBlock({ curl }: { curl: string }) { + const [copied, setCopied] = useState(false); + const copy = async () => { + await navigator.clipboard.writeText(curl); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + }; + return ( +
+
+ Reproduce with curl + +
+ 
+        {curl}
+
+
+ ); +} + +function stringify(value: unknown): string { + return JSON.stringify(value, null, 2); +} + +function prettyJson(raw: string): string { + try { + return JSON.stringify(JSON.parse(raw), null, 2); + } catch { + return raw; + } +} diff --git a/apps/dev-playground/server/index.ts b/apps/dev-playground/server/index.ts index 91179dacd..2aea01e94 100644 --- a/apps/dev-playground/server/index.ts +++ b/apps/dev-playground/server/index.ts @@ -14,6 +14,7 @@ import { import { WorkspaceClient } from "@databricks/sdk-experimental"; // TODO: re-enable once vector-search is exported from @databricks/appkit // import { vectorSearch } from "@databricks/appkit"; +import { internalTelemetryDebug } from "./internal-telemetry-debug-plugin"; import { lakebaseExamples } from "./lakebase-examples-plugin"; import { reconnect } from "./reconnect-plugin"; import { telemetryExamples } from "./telemetry-example-plugin"; @@ -54,6 +55,7 @@ createApp({ server(), reconnect(), telemetryExamples(), + internalTelemetryDebug(), analytics({}), genie({ spaces: { demo: process.env.DATABRICKS_GENIE_SPACE_ID ?? "placeholder" }, diff --git a/apps/dev-playground/server/internal-telemetry-debug-plugin.ts b/apps/dev-playground/server/internal-telemetry-debug-plugin.ts new file mode 100644 index 000000000..52edf7d2c --- /dev/null +++ b/apps/dev-playground/server/internal-telemetry-debug-plugin.ts @@ -0,0 +1,120 @@ +import { + Plugin, + type PluginManifest, + TelemetryReporter, + type TelemetrySendRequest, + type TelemetrySendResult, + toPlugin, +} from "@databricks/appkit"; +import type { Request, Response, Router } from "express"; + +type ReporterAction = "sendStartup" | "sendHeartbeat" | "flushRequestMetrics"; + +class InternalTelemetryDebug extends Plugin { + static manifest = { + name: "internal-telemetry-debug", + displayName: "Internal Telemetry Debug Plugin", + description: "Manually trigger internal telemetry events for testing", + resources: { required: [], optional: [] }, + } satisfies PluginManifest<"internal-telemetry-debug">; + + injectRoutes(router: Router): void { + router.post("/startup", this.handle("sendStartup")); + router.post("/heartbeat", this.handle("sendHeartbeat")); + router.post("/request-metrics-flush", this.handle("flushRequestMetrics")); + router.post("/request-metrics-record", (req, res) => { + const reporter = TelemetryReporter.getInstance(); + if (!reporter) { + res.status(503).json({ error: "Telemetry reporter not initialized" }); + return; + } + const { + method = "GET", + endpoint = "/api/internal-telemetry-debug/sample", + statusCode = 200, + latencyMs = 12, + } = (req.body ?? {}) as { + method?: string; + endpoint?: string; + statusCode?: number; + latencyMs?: number; + }; + reporter.recordRequest(method, endpoint, statusCode, latencyMs); + res.json({ + ok: true, + recorded: { method, endpoint, statusCode, latencyMs }, + }); + }); + } + + private handle(action: ReporterAction) { + return async (_req: Request, res: Response) => { + const reporter = TelemetryReporter.getInstance(); + if (!reporter) { + res.status(503).json({ error: "Telemetry reporter not initialized" }); + return; + } + try { + const result = await reporter[action](); + res.json(formatSuccess(action, result)); + } catch (error) { + res.status(500).json({ + ok: false, + action, + error: error instanceof Error ? error.message : String(error), + }); + } + }; + } +} + +const SENSITIVE_HEADERS = new Set(["authorization", "cookie", "set-cookie"]); + +function redactHeaders( + headers: Record, +): Record { + const out: Record = {}; + for (const [name, value] of Object.entries(headers)) { + out[name] = SENSITIVE_HEADERS.has(name.toLowerCase()) + ? "[REDACTED]" + : value; + } + return out; +} + +function formatSuccess( + action: ReporterAction, + result: TelemetrySendResult | null, +) { + if (!result) { + return { + ok: true, + action, + message: + "Nothing to send (request metrics buffer empty — record some first).", + }; + } + const safeRequest: TelemetrySendRequest = { + ...result.request, + headers: redactHeaders(result.request.headers), + }; + return { + ok: result.response.status >= 200 && result.response.status < 300, + action, + request: safeRequest, + response: result.response, + curl: toCurl(safeRequest), + }; +} + +function toCurl(req: TelemetrySendRequest): string { + const quote = (s: string) => s.replace(/'/g, "'\\''"); + const lines = [`curl -X POST '${quote(req.url)}'`]; + for (const [name, value] of Object.entries(req.headers)) { + lines.push(` -H '${quote(name)}: ${quote(value)}'`); + } + lines.push(` --data '${quote(req.body)}'`); + return lines.join(" \\\n"); +} + +export const internalTelemetryDebug = toPlugin(InternalTelemetryDebug); diff --git a/docs/docs/api/appkit/Class.TelemetryReporter.md b/docs/docs/api/appkit/Class.TelemetryReporter.md new file mode 100644 index 000000000..32ed511b8 --- /dev/null +++ b/docs/docs/api/appkit/Class.TelemetryReporter.md @@ -0,0 +1,116 @@ +# Class: TelemetryReporter + +## Methods + +### flushRequestMetrics() + +```ts +flushRequestMetrics(): Promise; +``` + +#### Returns + +`Promise`\<[`TelemetrySendResult`](Interface.TelemetrySendResult.md) \| `null`\> + +*** + +### recordRequest() + +```ts +recordRequest( + method: string, + routeTemplate: string, + statusCode: number, + latencyMs: number): void; +``` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `method` | `string` | +| `routeTemplate` | `string` | +| `statusCode` | `number` | +| `latencyMs` | `number` | + +#### Returns + +`void` + +*** + +### sendHeartbeat() + +```ts +sendHeartbeat(): Promise; +``` + +#### Returns + +`Promise`\<[`TelemetrySendResult`](Interface.TelemetrySendResult.md) \| `null`\> + +*** + +### sendStartup() + +```ts +sendStartup(): Promise; +``` + +#### Returns + +`Promise`\<[`TelemetrySendResult`](Interface.TelemetrySendResult.md) \| `null`\> + +*** + +### start() + +```ts +start(): void; +``` + +#### Returns + +`void` + +*** + +### stop() + +```ts +stop(): void; +``` + +#### Returns + +`void` + +*** + +### getInstance() + +```ts +static getInstance(): TelemetryReporter | null; +``` + +#### Returns + +`TelemetryReporter` \| `null` + +*** + +### initialize() + +```ts +static initialize(opts: ReporterOptions): TelemetryReporter; +``` + +#### Parameters + +| Parameter | Type | +| ------ | ------ | +| `opts` | `ReporterOptions` | + +#### Returns + +`TelemetryReporter` diff --git a/docs/docs/api/appkit/Function.createApp.md b/docs/docs/api/appkit/Function.createApp.md index 6a0b7cb2a..85cb584b0 100644 --- a/docs/docs/api/appkit/Function.createApp.md +++ b/docs/docs/api/appkit/Function.createApp.md @@ -4,6 +4,7 @@ function createApp(config: { cache?: CacheConfig; client?: WorkspaceClient; + disableInternalTelemetry?: boolean; onPluginsReady?: (appkit: PluginMap) => void | Promise; plugins?: T; telemetry?: TelemetryConfig; @@ -30,9 +31,10 @@ with an `asUser(req)` method for user-scoped execution. | Parameter | Type | | ------ | ------ | -| `config` | \{ `cache?`: [`CacheConfig`](Interface.CacheConfig.md); `client?`: `WorkspaceClient`; `onPluginsReady?`: (`appkit`: `PluginMap`\<`T`\>) => `void` \| `Promise`\<`void`\>; `plugins?`: `T`; `telemetry?`: [`TelemetryConfig`](Interface.TelemetryConfig.md); \} | +| `config` | \{ `cache?`: [`CacheConfig`](Interface.CacheConfig.md); `client?`: `WorkspaceClient`; `disableInternalTelemetry?`: `boolean`; `onPluginsReady?`: (`appkit`: `PluginMap`\<`T`\>) => `void` \| `Promise`\<`void`\>; `plugins?`: `T`; `telemetry?`: [`TelemetryConfig`](Interface.TelemetryConfig.md); \} | | `config.cache?` | [`CacheConfig`](Interface.CacheConfig.md) | | `config.client?` | `WorkspaceClient` | +| `config.disableInternalTelemetry?` | `boolean` | | `config.onPluginsReady?` | (`appkit`: `PluginMap`\<`T`\>) => `void` \| `Promise`\<`void`\> | | `config.plugins?` | `T` | | `config.telemetry?` | [`TelemetryConfig`](Interface.TelemetryConfig.md) | diff --git a/docs/docs/api/appkit/Interface.TelemetrySendRequest.md b/docs/docs/api/appkit/Interface.TelemetrySendRequest.md new file mode 100644 index 000000000..2f4726a2a --- /dev/null +++ b/docs/docs/api/appkit/Interface.TelemetrySendRequest.md @@ -0,0 +1,33 @@ +# Interface: TelemetrySendRequest + +## Properties + +### body + +```ts +body: string; +``` + +*** + +### headers + +```ts +headers: Record; +``` + +*** + +### method + +```ts +method: "POST"; +``` + +*** + +### url + +```ts +url: string; +``` diff --git a/docs/docs/api/appkit/Interface.TelemetrySendResponse.md b/docs/docs/api/appkit/Interface.TelemetrySendResponse.md new file mode 100644 index 000000000..49b72cb19 --- /dev/null +++ b/docs/docs/api/appkit/Interface.TelemetrySendResponse.md @@ -0,0 +1,25 @@ +# Interface: TelemetrySendResponse + +## Properties + +### body + +```ts +body: string; +``` + +*** + +### status + +```ts +status: number; +``` + +*** + +### statusText + +```ts +statusText: string; +``` diff --git a/docs/docs/api/appkit/Interface.TelemetrySendResult.md b/docs/docs/api/appkit/Interface.TelemetrySendResult.md new file mode 100644 index 000000000..9d49ba2c0 --- /dev/null +++ b/docs/docs/api/appkit/Interface.TelemetrySendResult.md @@ -0,0 +1,17 @@ +# Interface: TelemetrySendResult + +## Properties + +### request + +```ts +request: TelemetrySendRequest; +``` + +*** + +### response + +```ts +response: TelemetrySendResponse; +``` diff --git a/docs/docs/api/appkit/index.md b/docs/docs/api/appkit/index.md index 5a21e935f..e5a614478 100644 --- a/docs/docs/api/appkit/index.md +++ b/docs/docs/api/appkit/index.md @@ -24,6 +24,7 @@ plugin architecture, and React integration. | [PolicyDeniedError](Class.PolicyDeniedError.md) | Thrown when a policy denies an action. | | [ResourceRegistry](Class.ResourceRegistry.md) | Central registry for tracking plugin resource requirements. Deduplication uses type + resourceKey (machine-stable); alias is for display only. | | [ServerError](Class.ServerError.md) | Error thrown when server lifecycle operations fail. Use for server start/stop issues, configuration conflicts, etc. | +| [TelemetryReporter](Class.TelemetryReporter.md) | - | | [TunnelError](Class.TunnelError.md) | Error thrown when remote tunnel operations fail. Use for tunnel connection issues, message parsing failures, etc. | | [ValidationError](Class.ValidationError.md) | Error thrown when input validation fails. Use for invalid parameters, missing required fields, or type mismatches. | @@ -54,6 +55,9 @@ plugin architecture, and React integration. | [ServingEndpointRegistry](Interface.ServingEndpointRegistry.md) | Registry interface for serving endpoint type generation. Empty by default — augmented by the Vite type generator's `.d.ts` output via module augmentation. When populated, provides autocomplete for alias names and typed request/response/chunk per endpoint. | | [StreamExecutionSettings](Interface.StreamExecutionSettings.md) | Execution settings for streaming endpoints. Extends PluginExecutionSettings with SSE stream configuration. | | [TelemetryConfig](Interface.TelemetryConfig.md) | OpenTelemetry configuration for AppKit applications | +| [TelemetrySendRequest](Interface.TelemetrySendRequest.md) | - | +| [TelemetrySendResponse](Interface.TelemetrySendResponse.md) | - | +| [TelemetrySendResult](Interface.TelemetrySendResult.md) | - | | [ValidationResult](Interface.ValidationResult.md) | Result of validating all registered resources against the environment. | ## Type Aliases diff --git a/docs/docs/api/appkit/typedoc-sidebar.ts b/docs/docs/api/appkit/typedoc-sidebar.ts index 162c3e68b..6f61a14df 100644 --- a/docs/docs/api/appkit/typedoc-sidebar.ts +++ b/docs/docs/api/appkit/typedoc-sidebar.ts @@ -71,6 +71,11 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Class.ServerError", label: "ServerError" }, + { + type: "doc", + id: "api/appkit/Class.TelemetryReporter", + label: "TelemetryReporter" + }, { type: "doc", id: "api/appkit/Class.TunnelError", @@ -202,6 +207,21 @@ const typedocSidebar: SidebarsConfig = { id: "api/appkit/Interface.TelemetryConfig", label: "TelemetryConfig" }, + { + type: "doc", + id: "api/appkit/Interface.TelemetrySendRequest", + label: "TelemetrySendRequest" + }, + { + type: "doc", + id: "api/appkit/Interface.TelemetrySendResponse", + label: "TelemetrySendResponse" + }, + { + type: "doc", + id: "api/appkit/Interface.TelemetrySendResult", + label: "TelemetrySendResult" + }, { type: "doc", id: "api/appkit/Interface.ValidationResult", diff --git a/docs/docs/faq.md b/docs/docs/faq.md index 41667cae0..14700ec97 100644 --- a/docs/docs/faq.md +++ b/docs/docs/faq.md @@ -1,5 +1,5 @@ --- -sidebar_position: 6 +sidebar_position: 8 --- # FAQ diff --git a/docs/docs/internal-telemetry.mdx b/docs/docs/internal-telemetry.mdx new file mode 100644 index 000000000..844dbb981 --- /dev/null +++ b/docs/docs/internal-telemetry.mdx @@ -0,0 +1,76 @@ +--- +sidebar_position: 7 +--- + +# Internal telemetry + +AppKit sends a small amount of anonymized telemetry to Databricks so the +team can understand how the SDK is used and prioritize improvements. This +page documents exactly what is sent, when, and how to turn it off. + +## What we collect + +Every event is a single `AppkitLog` record with three top-level fields: + +| Field | Type | Source | +| ---------------- | ------ | ----------------------------------- | +| `event_name` | enum | One of `APP_STARTUP`, `HEARTBEAT`, `REQUEST_METRICS` | +| `app_id` | string | The app's OAuth client UUID (`DATABRICKS_CLIENT_ID`) | +| `appkit_version` | string | The AppKit SDK version | + +Each event also carries one of three event-specific bodies: + +- **`APP_STARTUP`** — emitted once when `createApp` finishes booting. The + body is an empty marker. +- **`HEARTBEAT`** — emitted every five minutes from a running app. Body is + an empty marker. +- **`REQUEST_METRICS`** — emitted once per minute, one record per HTTP + endpoint that received traffic in the window. Each record contains: + - `endpoint` — the route template (e.g. `GET /api/genie/:space_id/messages`), + never the raw request URL or any user-provided values. + - `request_count` + - `request_latency_ms_avg` + - `response_count_http4xx` + - `response_count_http5xx` + +That is the entire payload. AppKit does not send request bodies, response +bodies, headers, query parameters, user identifiers, or any other content +of your traffic. + +## How it's sent + +Events are POSTed to the workspace's authenticated `/telemetry-ext` +endpoint, signed with your app's service-principal token. Sending happens +fire-and-forget — every send is wrapped in a catch so that a failure in +the telemetry path can never affect a running app. + +## How to disable + +Two ways: + +**Code (per `createApp` call):** + +```ts +import { createApp, server } from "@databricks/appkit"; + +await createApp({ + plugins: [server()], + disableInternalTelemetry: true, +}); +``` + +**Environment (workspace-wide):** + +```sh +DISABLE_APPKIT_INTERNAL_TELEMETRY=true +``` + +Either one fully disables the reporter — no events are emitted and no +network calls are made. + +## Inspecting events locally + +The dev-playground ships an **Internal Telemetry** tab that lets you +trigger each event type on demand and inspect the exact request, the +response, and a `curl` command you can replay. Use it to verify what your +deployed app would send before enabling telemetry in production. diff --git a/packages/appkit/src/core/appkit.ts b/packages/appkit/src/core/appkit.ts index 607a15524..f7ac99bed 100644 --- a/packages/appkit/src/core/appkit.ts +++ b/packages/appkit/src/core/appkit.ts @@ -8,8 +8,13 @@ import type { PluginData, PluginMap, } from "shared"; +import { version as productVersion } from "../../package.json"; import { CacheManager } from "../cache"; import { ServiceContext } from "../context"; +import { + isInternalTelemetryEnabled, + TelemetryReporter, +} from "../internal-telemetry"; import { createLogger } from "../logging/logger"; import { ResourceRegistry, ResourceType } from "../registry"; import type { TelemetryConfig } from "../telemetry"; @@ -171,6 +176,7 @@ export class AppKit { cache?: CacheConfig; client?: WorkspaceClient; onPluginsReady?: (appkit: PluginMap) => void | Promise; + disableInternalTelemetry?: boolean; } = {}, ): Promise> { // Initialize core services @@ -212,6 +218,10 @@ export class AppKit { logger.debug("onPluginsReady hook completed"); } + if (isInternalTelemetryEnabled(config)) { + AppKit.bootstrapInternalTelemetry(); + } + const serverPlugin = instance.#pluginInstances.server; if (serverPlugin && typeof (serverPlugin as any).start === "function") { await (serverPlugin as any).start(); @@ -220,6 +230,20 @@ export class AppKit { return handle; } + private static bootstrapInternalTelemetry(): void { + const serviceCtx = ServiceContext.get(); + const reporter = TelemetryReporter.initialize({ + workspaceHost: + serviceCtx.client.config?.host || process.env.DATABRICKS_HOST || "", + workspaceId: serviceCtx.workspaceId, + client: serviceCtx.client, + appId: process.env.DATABRICKS_CLIENT_ID || "", + appkitVersion: productVersion, + }); + reporter.start(); + reporter.sendStartup().catch(() => {}); + } + private static preparePlugins( plugins: PluginData[], ) { @@ -279,6 +303,7 @@ export async function createApp< cache?: CacheConfig; client?: WorkspaceClient; onPluginsReady?: (appkit: PluginMap) => void | Promise; + disableInternalTelemetry?: boolean; } = {}, ): Promise> { return AppKit._createApp(config); diff --git a/packages/appkit/src/core/tests/databricks.test.ts b/packages/appkit/src/core/tests/databricks.test.ts index c05345a6b..948502958 100644 --- a/packages/appkit/src/core/tests/databricks.test.ts +++ b/packages/appkit/src/core/tests/databricks.test.ts @@ -6,6 +6,24 @@ import type { PluginManifest } from "../../registry/types"; import { ResourceType } from "../../registry/types"; import { AppKit, createApp } from "../appkit"; +const mockReporter = { + start: vi.fn(), + stop: vi.fn(), + sendStartup: vi.fn().mockResolvedValue(undefined), + sendHeartbeat: vi.fn().mockResolvedValue(undefined), + flushRequestMetrics: vi.fn().mockResolvedValue(undefined), + recordRequest: vi.fn(), +}; + +vi.mock("../../internal-telemetry", () => ({ + isInternalTelemetryEnabled: vi.fn().mockReturnValue(true), + TelemetryReporter: { + initialize: vi.fn(() => mockReporter), + getInstance: vi.fn(() => mockReporter), + _reset: vi.fn(), + }, +})); + // Generic test manifest for test plugins const createTestManifest = (name: string): PluginManifest => ({ name, @@ -630,6 +648,60 @@ describe("AppKit", () => { }); }); + describe("internal telemetry", () => { + test("initializes the reporter and fires sendStartup after createApp", async () => { + const { TelemetryReporter } = await import("../../internal-telemetry"); + mockReporter.sendStartup.mockClear(); + mockReporter.start.mockClear(); + vi.mocked(TelemetryReporter.initialize).mockClear(); + + await createApp({ + plugins: [{ plugin: CoreTestPlugin, config: {}, name: "coreTest" }], + }); + + // Allow the fire-and-forget promise chain to resolve + await new Promise((r) => setTimeout(r, 10)); + + expect(TelemetryReporter.initialize).toHaveBeenCalledWith( + expect.objectContaining({ + appkitVersion: expect.any(String), + client: expect.anything(), + }), + ); + expect(mockReporter.start).toHaveBeenCalledOnce(); + expect(mockReporter.sendStartup).toHaveBeenCalledOnce(); + }); + + test("skips bootstrap when isInternalTelemetryEnabled returns false", async () => { + const { isInternalTelemetryEnabled, TelemetryReporter } = await import( + "../../internal-telemetry" + ); + vi.mocked(TelemetryReporter.initialize).mockClear(); + mockReporter.sendStartup.mockClear(); + vi.mocked(isInternalTelemetryEnabled).mockReturnValue(false); + + await createApp({ plugins: [] }); + + await new Promise((r) => setTimeout(r, 10)); + + expect(TelemetryReporter.initialize).not.toHaveBeenCalled(); + expect(mockReporter.sendStartup).not.toHaveBeenCalled(); + vi.mocked(isInternalTelemetryEnabled).mockReturnValue(true); + }); + + test("does not crash startup if sendStartup rejects", async () => { + mockReporter.sendStartup.mockRejectedValueOnce( + new Error("telemetry failure"), + ); + + const instance = await createApp({ + plugins: [{ plugin: CoreTestPlugin, config: {}, name: "coreTest" }], + }); + + expect(instance).toBeDefined(); + }); + }); + describe("SDK context binding", () => { test("should bind SDK methods to plugin instance", async () => { class ContextTestPlugin implements BasePlugin { diff --git a/packages/appkit/src/index.ts b/packages/appkit/src/index.ts index eecda8e3d..1078a6ea9 100644 --- a/packages/appkit/src/index.ts +++ b/packages/appkit/src/index.ts @@ -47,6 +47,12 @@ export { TunnelError, ValidationError, } from "./errors"; +export { + TelemetryReporter, + type TelemetrySendRequest, + type TelemetrySendResponse, + type TelemetrySendResult, +} from "./internal-telemetry"; // Plugin authoring export { type ExecutionResult, diff --git a/packages/appkit/src/internal-telemetry/appkit-log.ts b/packages/appkit/src/internal-telemetry/appkit-log.ts new file mode 100644 index 000000000..1e857d683 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/appkit-log.ts @@ -0,0 +1,62 @@ +// IMPORTANT: keep this file in sync with the AppkitLog proto schema served by +// the Databricks client telemetry endpoint. Field names use proto JSON +// conventions (snake_case) so the wire format matches the backend. + +export type AppkitEventName = + | "APPKIT_EVENT_NAME_UNSPECIFIED" + | "APP_STARTUP" + | "HEARTBEAT" + | "REQUEST_METRICS"; + +export interface AppStartupEvent { + placeholder?: boolean; +} + +export interface HeartbeatEvent { + placeholder?: boolean; +} + +export interface RequestMetricsEvent { + endpoint?: string; + request_count?: number; + request_latency_ms_avg?: number; + response_count_http4xx?: number; + response_count_http5xx?: number; +} + +export interface AppkitLog { + event_name: AppkitEventName; + app_id?: string; + appkit_version?: string; + app_startup_event?: AppStartupEvent; + heartbeat_event?: HeartbeatEvent; + request_metrics_event?: RequestMetricsEvent; +} + +interface AppkitLogEnvelope { + frontend_log_event_id: string; + inferred_timestamp_millis: number; + entry: { appkit_log: AppkitLog }; +} + +interface TelemetryPayload { + uploadTime: number; + items: never[]; + protoLogs: string[]; +} + +export function wrapAppkitLog(log: AppkitLog): AppkitLogEnvelope { + return { + frontend_log_event_id: `appkit-${log.event_name.toLowerCase()}-${crypto.randomUUID()}`, + inferred_timestamp_millis: Date.now(), + entry: { appkit_log: log }, + }; +} + +export function buildAppkitPayload(logs: AppkitLog[]): TelemetryPayload { + return { + uploadTime: Date.now(), + items: [], + protoLogs: logs.map((log) => JSON.stringify(wrapAppkitLog(log))), + }; +} diff --git a/packages/appkit/src/internal-telemetry/client.ts b/packages/appkit/src/internal-telemetry/client.ts new file mode 100644 index 000000000..17869889f --- /dev/null +++ b/packages/appkit/src/internal-telemetry/client.ts @@ -0,0 +1,85 @@ +import type { WorkspaceClient } from "@databricks/sdk-experimental"; + +const TIMEOUT_MS = 10_000; + +export interface TelemetrySendRequest { + url: string; + method: "POST"; + headers: Record; + body: string; +} + +export interface TelemetrySendResponse { + status: number; + statusText: string; + body: string; +} + +export interface TelemetrySendResult { + request: TelemetrySendRequest; + response: TelemetrySendResponse; +} + +function normalizeHost(rawHost: string): string { + const host = rawHost.replace(/\/+$/, ""); + if (!host) return ""; + return host.startsWith("http") ? host : `https://${host}`; +} + +function headersToObject(h: Headers): Record { + const out: Record = {}; + h.forEach((value, key) => { + out[key] = value; + }); + return out; +} + +/** + * Authenticated POST to the Databricks Client Telemetry endpoint. + * Returns the dispatched request and the received response so callers can + * surface them for debugging. Throws on network, auth, or misconfiguration + * errors; HTTP-level failures (4xx/5xx) are returned as-is on `response`. + */ +export async function postTelemetry(params: { + workspaceHost: string; + workspaceId: string; + client: WorkspaceClient; + payload: object; +}): Promise { + const host = normalizeHost(params.workspaceHost); + if (!host) throw new Error("Telemetry: workspaceHost is empty"); + if (!params.workspaceId) throw new Error("Telemetry: workspaceId is empty"); + + const url = `${host}/telemetry-ext?o=${params.workspaceId}`; + const body = JSON.stringify(params.payload); + + const headers = new Headers({ + "Content-Type": "application/json", + "X-Databricks-Org-Id": params.workspaceId, + }); + await params.client.config.authenticate(headers); + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); + + try { + const response = await fetch(url, { + method: "POST", + headers, + body, + signal: controller.signal, + redirect: "manual", + }); + const responseBody = await response.text(); + return { + request: { url, method: "POST", headers: headersToObject(headers), body }, + response: { + status: response.status, + statusText: response.statusText, + body: responseBody, + }, + }; + } finally { + clearTimeout(timer); + } +} diff --git a/packages/appkit/src/internal-telemetry/config.ts b/packages/appkit/src/internal-telemetry/config.ts new file mode 100644 index 000000000..b902359c9 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/config.ts @@ -0,0 +1,11 @@ +/** + * Checks whether internal telemetry is enabled. + * Shared across all telemetry event types (startup, heartbeat, metrics, etc.). + */ +export function isInternalTelemetryEnabled(opts?: { + disableInternalTelemetry?: boolean; +}): boolean { + if (opts?.disableInternalTelemetry) return false; + if (process.env.DISABLE_APPKIT_INTERNAL_TELEMETRY === "true") return false; + return true; +} diff --git a/packages/appkit/src/internal-telemetry/index.ts b/packages/appkit/src/internal-telemetry/index.ts new file mode 100644 index 000000000..a2bc503d8 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/index.ts @@ -0,0 +1,13 @@ +// Internal telemetry: APP_STARTUP, HEARTBEAT, and REQUEST_METRICS events +// POSTed to /telemetry-ext so the Databricks team can prioritize SDK work. +// Disable with disableInternalTelemetry: true on createApp, or +// DISABLE_APPKIT_INTERNAL_TELEMETRY=true. +// Full data inventory: docs/docs/internal-telemetry.mdx. + +export type { + TelemetrySendRequest, + TelemetrySendResponse, + TelemetrySendResult, +} from "./client.js"; +export { isInternalTelemetryEnabled } from "./config.js"; +export { TelemetryReporter } from "./reporter.js"; diff --git a/packages/appkit/src/internal-telemetry/reporter.ts b/packages/appkit/src/internal-telemetry/reporter.ts new file mode 100644 index 000000000..68b34e561 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/reporter.ts @@ -0,0 +1,194 @@ +import type { WorkspaceClient } from "@databricks/sdk-experimental"; +import { + type AppkitLog, + buildAppkitPayload, + type RequestMetricsEvent, +} from "./appkit-log.js"; +import { postTelemetry, type TelemetrySendResult } from "./client.js"; + +const DEFAULT_HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000; +const DEFAULT_METRICS_FLUSH_INTERVAL_MS = 60 * 1000; + +interface ReporterOptions { + workspaceHost: string; + workspaceId: Promise | string; + client: WorkspaceClient; + appId: string; + appkitVersion: string; + heartbeatIntervalMs?: number; + metricsFlushIntervalMs?: number; +} + +interface RequestBucket { + count: number; + latencyMsTotal: number; + http4xx: number; + http5xx: number; +} + +function envIntervalMs(name: string, fallback: number): number { + const raw = process.env[name]; + if (!raw) return fallback; + const n = Number(raw); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +export class TelemetryReporter { + static #instance: TelemetryReporter | null = null; + + readonly #host: string; + readonly #workspaceIdPromise: Promise; + readonly #client: WorkspaceClient; + readonly #appId: string; + readonly #appkitVersion: string; + readonly #heartbeatIntervalMs: number; + readonly #metricsFlushIntervalMs: number; + + #heartbeatTimer: NodeJS.Timeout | null = null; + #metricsTimer: NodeJS.Timeout | null = null; + #buckets: Map = new Map(); + + private constructor(opts: ReporterOptions) { + this.#host = opts.workspaceHost; + this.#workspaceIdPromise = Promise.resolve(opts.workspaceId); + // Mark the rejection (if any) as handled so a misconfigured workspaceId + // doesn't trigger an unhandled-rejection warning before the first #send + // awaits it. The original promise still rejects when awaited. + this.#workspaceIdPromise.catch(() => {}); + this.#client = opts.client; + this.#appId = opts.appId; + this.#appkitVersion = opts.appkitVersion; + this.#heartbeatIntervalMs = + opts.heartbeatIntervalMs ?? + envIntervalMs( + "APPKIT_TELEMETRY_HEARTBEAT_INTERVAL_MS", + DEFAULT_HEARTBEAT_INTERVAL_MS, + ); + this.#metricsFlushIntervalMs = + opts.metricsFlushIntervalMs ?? + envIntervalMs( + "APPKIT_TELEMETRY_METRICS_FLUSH_INTERVAL_MS", + DEFAULT_METRICS_FLUSH_INTERVAL_MS, + ); + } + + static initialize(opts: ReporterOptions): TelemetryReporter { + TelemetryReporter.#instance?.stop(); + TelemetryReporter.#instance = new TelemetryReporter(opts); + return TelemetryReporter.#instance; + } + + static getInstance(): TelemetryReporter | null { + return TelemetryReporter.#instance; + } + + /** @internal Test-only reset. */ + static _reset(): void { + TelemetryReporter.#instance?.stop(); + TelemetryReporter.#instance = null; + } + + start(): void { + if (this.#heartbeatTimer || this.#metricsTimer) return; + this.#heartbeatTimer = setInterval(() => { + this.sendHeartbeat().catch(() => {}); + }, this.#heartbeatIntervalMs); + this.#heartbeatTimer.unref?.(); + + this.#metricsTimer = setInterval(() => { + this.flushRequestMetrics().catch(() => {}); + }, this.#metricsFlushIntervalMs); + this.#metricsTimer.unref?.(); + } + + stop(): void { + if (this.#heartbeatTimer) clearInterval(this.#heartbeatTimer); + if (this.#metricsTimer) clearInterval(this.#metricsTimer); + this.#heartbeatTimer = null; + this.#metricsTimer = null; + } + + recordRequest( + method: string, + routeTemplate: string, + statusCode: number, + latencyMs: number, + ): void { + if (!routeTemplate) return; + const key = `${method.toUpperCase()} ${routeTemplate}`; + const bucket = this.#buckets.get(key) ?? { + count: 0, + latencyMsTotal: 0, + http4xx: 0, + http5xx: 0, + }; + bucket.count += 1; + bucket.latencyMsTotal += Math.max(0, latencyMs); + if (statusCode >= 400 && statusCode < 500) bucket.http4xx += 1; + if (statusCode >= 500 && statusCode < 600) bucket.http5xx += 1; + this.#buckets.set(key, bucket); + } + + async sendStartup(): Promise { + return this.#send([ + this.#wrap({ + event_name: "APP_STARTUP", + app_startup_event: { placeholder: true }, + }), + ]); + } + + async sendHeartbeat(): Promise { + return this.#send([ + this.#wrap({ + event_name: "HEARTBEAT", + heartbeat_event: { placeholder: true }, + }), + ]); + } + + async flushRequestMetrics(): Promise { + if (this.#buckets.size === 0) return null; + const drained = this.#buckets; + this.#buckets = new Map(); + + const logs: AppkitLog[] = []; + for (const [endpoint, bucket] of drained) { + const event: RequestMetricsEvent = { + endpoint, + request_count: bucket.count, + request_latency_ms_avg: Math.round( + bucket.latencyMsTotal / bucket.count, + ), + response_count_http4xx: bucket.http4xx, + response_count_http5xx: bucket.http5xx, + }; + logs.push( + this.#wrap({ + event_name: "REQUEST_METRICS", + request_metrics_event: event, + }), + ); + } + return this.#send(logs); + } + + #wrap(partial: AppkitLog): AppkitLog { + return { + ...partial, + app_id: this.#appId, + appkit_version: this.#appkitVersion, + }; + } + + async #send(logs: AppkitLog[]): Promise { + if (logs.length === 0) return null; + const workspaceId = await this.#workspaceIdPromise; + return postTelemetry({ + workspaceHost: this.#host, + workspaceId, + client: this.#client, + payload: buildAppkitPayload(logs), + }); + } +} diff --git a/packages/appkit/src/internal-telemetry/tests/appkit-log.test.ts b/packages/appkit/src/internal-telemetry/tests/appkit-log.test.ts new file mode 100644 index 000000000..4ef5286a2 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/tests/appkit-log.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from "vitest"; +import { buildAppkitPayload, wrapAppkitLog } from "../appkit-log"; + +describe("appkit-log", () => { + test("wrapAppkitLog produces a typed envelope", () => { + const envelope = wrapAppkitLog({ + event_name: "HEARTBEAT", + app_id: "id", + appkit_version: "1.0.0", + heartbeat_event: { placeholder: true }, + }); + expect(envelope.frontend_log_event_id).toMatch(/^appkit-heartbeat-/); + expect(envelope.entry.appkit_log.event_name).toBe("HEARTBEAT"); + expect(typeof envelope.inferred_timestamp_millis).toBe("number"); + }); + + test("buildAppkitPayload encodes one protoLog per log", () => { + const payload = buildAppkitPayload([ + { event_name: "APP_STARTUP", app_startup_event: { placeholder: true } }, + { event_name: "HEARTBEAT", heartbeat_event: { placeholder: true } }, + ]); + expect(payload.items).toEqual([]); + expect(payload.protoLogs).toHaveLength(2); + expect(JSON.parse(payload.protoLogs[0]).entry.appkit_log.event_name).toBe( + "APP_STARTUP", + ); + }); +}); diff --git a/packages/appkit/src/internal-telemetry/tests/client.test.ts b/packages/appkit/src/internal-telemetry/tests/client.test.ts new file mode 100644 index 000000000..d2f958065 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/tests/client.test.ts @@ -0,0 +1,160 @@ +import type { WorkspaceClient } from "@databricks/sdk-experimental"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { postTelemetry } from "../client"; + +function createMockClient(): WorkspaceClient { + return { + config: { + authenticate: vi.fn(async (headers: Headers) => { + headers.set("Authorization", "Bearer mock-sp-token"); + }), + }, + } as unknown as WorkspaceClient; +} + +const samplePayload = { + uploadTime: 123, + items: [], + protoLogs: ['{"entry":{"appkit_log":{"event_name":"APP_STARTUP"}}}'], +}; + +const defaultOpts = () => ({ + workspaceHost: "https://my-workspace.cloud.databricks.com", + workspaceId: "1234567890", + client: createMockClient(), + payload: samplePayload, +}); + +let fetchSpy: ReturnType; + +beforeEach(() => { + fetchSpy = vi.fn().mockResolvedValue(new Response("", { status: 200 })); + vi.stubGlobal("fetch", fetchSpy); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +describe("postTelemetry", () => { + test("POSTs to the SP-friendly /telemetry-ext endpoint", async () => { + await postTelemetry(defaultOpts()); + + expect(fetchSpy).toHaveBeenCalledOnce(); + const [url, options] = fetchSpy.mock.calls[0]; + expect(url).toBe( + "https://my-workspace.cloud.databricks.com/telemetry-ext?o=1234567890", + ); + expect(options.method).toBe("POST"); + expect(options.redirect).toBe("manual"); + }); + + test("authenticates via WorkspaceClient and sets headers", async () => { + const opts = defaultOpts(); + await postTelemetry(opts); + + expect(opts.client.config.authenticate).toHaveBeenCalledOnce(); + const [, options] = fetchSpy.mock.calls[0]; + const headers = options.headers as Headers; + expect(headers.get("Content-Type")).toBe("application/json"); + expect(headers.get("X-Databricks-Org-Id")).toBe("1234567890"); + expect(headers.get("Authorization")).toBe("Bearer mock-sp-token"); + }); + + test("serializes the payload as JSON in the body", async () => { + await postTelemetry(defaultOpts()); + + const [, options] = fetchSpy.mock.calls[0]; + const body = JSON.parse(options.body); + expect(body).toEqual(samplePayload); + }); + + test("returns 3xx responses as-is (no automatic redirect follow)", async () => { + fetchSpy.mockResolvedValueOnce( + new Response("redirected", { + status: 302, + headers: { location: "/login.html?next_url=%2Ftelemetry-ext" }, + }), + ); + + const result = await postTelemetry(defaultOpts()); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(result.response.status).toBe(302); + expect(result.response.body).toBe("redirected"); + }); + + test("propagates fetch errors to the caller", async () => { + fetchSpy.mockRejectedValue(new Error("network failure")); + + await expect(postTelemetry(defaultOpts())).rejects.toThrow( + "network failure", + ); + }); + + test("returns 4xx/5xx responses without throwing", async () => { + fetchSpy.mockResolvedValue(new Response("boom", { status: 500 })); + + const result = await postTelemetry(defaultOpts()); + expect(result.response.status).toBe(500); + expect(result.response.body).toBe("boom"); + }); + + test("propagates authentication failures", async () => { + const opts = defaultOpts(); + ( + opts.client.config.authenticate as ReturnType + ).mockRejectedValue(new Error("auth failed")); + + await expect(postTelemetry(opts)).rejects.toThrow("auth failed"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("throws when workspaceHost is empty", async () => { + await expect( + postTelemetry({ ...defaultOpts(), workspaceHost: "" }), + ).rejects.toThrow(/workspaceHost/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("throws when workspaceId is empty", async () => { + await expect( + postTelemetry({ ...defaultOpts(), workspaceId: "" }), + ).rejects.toThrow(/workspaceId/); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("returns the dispatched request and response", async () => { + fetchSpy.mockResolvedValue(new Response("ok", { status: 200 })); + + const result = await postTelemetry(defaultOpts()); + expect(result.request.method).toBe("POST"); + expect(result.request.url).toBe( + "https://my-workspace.cloud.databricks.com/telemetry-ext?o=1234567890", + ); + expect(result.response.status).toBe(200); + expect(result.response.body).toBe("ok"); + }); + + test("normalizes a host without protocol", async () => { + await postTelemetry({ + ...defaultOpts(), + workspaceHost: "my-workspace.cloud.databricks.com", + }); + const [url] = fetchSpy.mock.calls[0]; + expect(url).toBe( + "https://my-workspace.cloud.databricks.com/telemetry-ext?o=1234567890", + ); + }); + + test("strips trailing slashes from the host", async () => { + await postTelemetry({ + ...defaultOpts(), + workspaceHost: "https://my-workspace.cloud.databricks.com///", + }); + const [url] = fetchSpy.mock.calls[0]; + expect(url).toBe( + "https://my-workspace.cloud.databricks.com/telemetry-ext?o=1234567890", + ); + }); +}); diff --git a/packages/appkit/src/internal-telemetry/tests/config.test.ts b/packages/appkit/src/internal-telemetry/tests/config.test.ts new file mode 100644 index 000000000..1d2e5f9f4 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/tests/config.test.ts @@ -0,0 +1,41 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import { isInternalTelemetryEnabled } from "../config"; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("isInternalTelemetryEnabled", () => { + test("returns true by default", () => { + expect(isInternalTelemetryEnabled()).toBe(true); + }); + + test("returns false when disableInternalTelemetry is true", () => { + expect(isInternalTelemetryEnabled({ disableInternalTelemetry: true })).toBe( + false, + ); + }); + + test("returns true when disableInternalTelemetry is false", () => { + expect( + isInternalTelemetryEnabled({ disableInternalTelemetry: false }), + ).toBe(true); + }); + + test("returns false when DISABLE_APPKIT_INTERNAL_TELEMETRY env var is true", () => { + vi.stubEnv("DISABLE_APPKIT_INTERNAL_TELEMETRY", "true"); + expect(isInternalTelemetryEnabled()).toBe(false); + }); + + test("returns true when DISABLE_APPKIT_INTERNAL_TELEMETRY env var is not true", () => { + vi.stubEnv("DISABLE_APPKIT_INTERNAL_TELEMETRY", "false"); + expect(isInternalTelemetryEnabled()).toBe(true); + }); + + test("config option takes precedence over env var", () => { + vi.stubEnv("DISABLE_APPKIT_INTERNAL_TELEMETRY", "false"); + expect(isInternalTelemetryEnabled({ disableInternalTelemetry: true })).toBe( + false, + ); + }); +}); diff --git a/packages/appkit/src/internal-telemetry/tests/reporter.test.ts b/packages/appkit/src/internal-telemetry/tests/reporter.test.ts new file mode 100644 index 000000000..f72b6c7f2 --- /dev/null +++ b/packages/appkit/src/internal-telemetry/tests/reporter.test.ts @@ -0,0 +1,217 @@ +import type { WorkspaceClient } from "@databricks/sdk-experimental"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { TelemetryReporter } from "../reporter"; + +function createMockClient(): WorkspaceClient { + return { + config: { + authenticate: vi.fn(async (headers: Headers) => { + headers.set("Authorization", "Bearer mock-sp-token"); + }), + }, + } as unknown as WorkspaceClient; +} + +const baseOpts = () => ({ + workspaceHost: "https://my-workspace.cloud.databricks.com", + workspaceId: "1234567890", + client: createMockClient(), + appId: "app-uuid-1", + appkitVersion: "0.27.0", + heartbeatIntervalMs: 1_000_000, + metricsFlushIntervalMs: 1_000_000, +}); + +let fetchSpy: ReturnType; + +beforeEach(() => { + fetchSpy = vi.fn().mockResolvedValue(new Response("", { status: 200 })); + vi.stubGlobal("fetch", fetchSpy); +}); + +afterEach(() => { + TelemetryReporter._reset(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +function lastProtoLog() { + const calls = fetchSpy.mock.calls; + const [, options] = calls[calls.length - 1]; + const body = JSON.parse(options.body as string); + return JSON.parse(body.protoLogs[0]); +} + +describe("TelemetryReporter", () => { + test("getInstance returns null before initialize", () => { + expect(TelemetryReporter.getInstance()).toBeNull(); + }); + + test("sendStartup emits an APP_STARTUP appkit_log", async () => { + const reporter = TelemetryReporter.initialize(baseOpts()); + await reporter.sendStartup(); + + const log = lastProtoLog(); + expect(log.entry.appkit_log).toMatchObject({ + event_name: "APP_STARTUP", + app_id: "app-uuid-1", + appkit_version: "0.27.0", + app_startup_event: { placeholder: true }, + }); + }); + + test("sendHeartbeat emits a HEARTBEAT appkit_log", async () => { + const reporter = TelemetryReporter.initialize(baseOpts()); + await reporter.sendHeartbeat(); + + const log = lastProtoLog(); + expect(log.entry.appkit_log).toMatchObject({ + event_name: "HEARTBEAT", + heartbeat_event: { placeholder: true }, + }); + }); + + test("recordRequest aggregates by method+route and flush sends one log per endpoint", async () => { + const reporter = TelemetryReporter.initialize(baseOpts()); + reporter.recordRequest("GET", "/api/x", 200, 100); + reporter.recordRequest("get", "/api/x", 200, 200); + reporter.recordRequest("GET", "/api/x", 500, 50); + reporter.recordRequest("POST", "/api/y", 404, 10); + + await reporter.flushRequestMetrics(); + + expect(fetchSpy).toHaveBeenCalledOnce(); + const [, options] = fetchSpy.mock.calls[0]; + const protoLogs = JSON.parse(options.body as string).protoLogs as string[]; + expect(protoLogs).toHaveLength(2); + + const events = protoLogs + .map((s) => JSON.parse(s).entry.appkit_log.request_metrics_event) + .sort((a, b) => a.endpoint.localeCompare(b.endpoint)); + + expect(events[0]).toEqual({ + endpoint: "GET /api/x", + request_count: 3, + request_latency_ms_avg: 117, // (100 + 200 + 50) / 3 = 116.67 -> 117 + response_count_http4xx: 0, + response_count_http5xx: 1, + }); + expect(events[1]).toEqual({ + endpoint: "POST /api/y", + request_count: 1, + request_latency_ms_avg: 10, + response_count_http4xx: 1, + response_count_http5xx: 0, + }); + }); + + test("flushRequestMetrics is a no-op when there are no buckets", async () => { + const reporter = TelemetryReporter.initialize(baseOpts()); + await reporter.flushRequestMetrics(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("flushRequestMetrics drains the aggregator after sending", async () => { + const reporter = TelemetryReporter.initialize(baseOpts()); + reporter.recordRequest("GET", "/api/x", 200, 10); + await reporter.flushRequestMetrics(); + fetchSpy.mockClear(); + await reporter.flushRequestMetrics(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("recordRequest ignores entries without a route template", async () => { + const reporter = TelemetryReporter.initialize(baseOpts()); + reporter.recordRequest("GET", "", 200, 10); + await reporter.flushRequestMetrics(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("start schedules heartbeat and metrics flush; stop clears them", () => { + vi.useFakeTimers(); + const reporter = TelemetryReporter.initialize({ + ...baseOpts(), + heartbeatIntervalMs: 1_000, + metricsFlushIntervalMs: 500, + }); + const heartbeatSpy = vi + .spyOn(reporter, "sendHeartbeat") + .mockResolvedValue(null); + const flushSpy = vi + .spyOn(reporter, "flushRequestMetrics") + .mockResolvedValue(null); + + reporter.start(); + vi.advanceTimersByTime(1_500); + expect(heartbeatSpy).toHaveBeenCalledTimes(1); + expect(flushSpy).toHaveBeenCalledTimes(3); + + reporter.stop(); + vi.advanceTimersByTime(5_000); + expect(heartbeatSpy).toHaveBeenCalledTimes(1); + expect(flushSpy).toHaveBeenCalledTimes(3); + vi.useRealTimers(); + }); + + test("propagates fetch errors so callers can surface them", async () => { + fetchSpy.mockRejectedValue(new Error("network down")); + const reporter = TelemetryReporter.initialize(baseOpts()); + await expect(reporter.sendHeartbeat()).rejects.toThrow("network down"); + }); + + test("propagates a rejecting workspaceId promise", async () => { + const reporter = TelemetryReporter.initialize({ + ...baseOpts(), + workspaceId: Promise.reject(new Error("nope")), + }); + await expect(reporter.sendHeartbeat()).rejects.toThrow("nope"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + test("interval timers swallow rejections silently", async () => { + vi.useFakeTimers(); + fetchSpy.mockRejectedValue(new Error("network down")); + const reporter = TelemetryReporter.initialize({ + ...baseOpts(), + heartbeatIntervalMs: 100, + metricsFlushIntervalMs: 1_000_000, + }); + reporter.start(); + await vi.advanceTimersByTimeAsync(150); + // No unhandled rejection means the timer's outer .catch worked. + reporter.stop(); + vi.useRealTimers(); + }); + + test("re-initialize stops the previous instance's timers", () => { + vi.useFakeTimers(); + const first = TelemetryReporter.initialize({ + ...baseOpts(), + heartbeatIntervalMs: 100, + metricsFlushIntervalMs: 100, + }); + const firstHeartbeat = vi + .spyOn(first, "sendHeartbeat") + .mockResolvedValue(null); + first.start(); + + TelemetryReporter.initialize({ + ...baseOpts(), + heartbeatIntervalMs: 1_000_000, + metricsFlushIntervalMs: 1_000_000, + }); + + vi.advanceTimersByTime(500); + // The first reporter's timers must have been cleared by the re-init. + expect(firstHeartbeat).not.toHaveBeenCalled(); + vi.useRealTimers(); + }); + + test("returns dispatched request and response from sendStartup", async () => { + fetchSpy.mockResolvedValue(new Response("ok", { status: 200 })); + const reporter = TelemetryReporter.initialize(baseOpts()); + const result = await reporter.sendStartup(); + expect(result?.request.method).toBe("POST"); + expect(result?.response.status).toBe(200); + }); +}); diff --git a/packages/appkit/src/plugins/server/index.ts b/packages/appkit/src/plugins/server/index.ts index 8ed13cea0..a72700537 100644 --- a/packages/appkit/src/plugins/server/index.ts +++ b/packages/appkit/src/plugins/server/index.ts @@ -5,6 +5,7 @@ import dotenv from "dotenv"; import express from "express"; import type { PluginClientConfigs, PluginPhase } from "shared"; import { ServerError } from "../../errors"; +import { TelemetryReporter } from "../../internal-telemetry"; import { createLogger } from "../../logging/logger"; import { Plugin, toPlugin } from "../../plugin"; import type { PluginManifest } from "../../registry"; @@ -96,6 +97,7 @@ export class ServerPlugin extends Plugin { * @returns The express application. */ async start(): Promise { + this.serverApplication.use(requestMetricsMiddleware); this.serverApplication.use( express.json({ type: (req) => { @@ -414,6 +416,29 @@ export class ServerPlugin extends Plugin { const EXCLUDED_PLUGINS: string[] = [ServerPlugin.manifest.name]; +function requestMetricsMiddleware( + req: express.Request, + res: express.Response, + next: express.NextFunction, +) { + const startMs = Date.now(); + res.on("finish", () => { + const reporter = TelemetryReporter.getInstance(); + if (!reporter) return; + const routePath = (req.route as { path?: string } | undefined)?.path; + if (!routePath) return; + const baseUrl = req.baseUrl ?? ""; + const template = `${baseUrl}${routePath}`; + reporter.recordRequest( + req.method, + template, + res.statusCode, + Date.now() - startMs, + ); + }); + next(); +} + /** * @internal */