diff --git a/docs/config.json b/docs/config.json index 23330cb22..9ae417c32 100644 --- a/docs/config.json +++ b/docs/config.json @@ -50,6 +50,10 @@ "label": "Agentic Cycle", "to": "guides/agentic-cycle" }, + { + "label": "Middleware", + "to": "guides/middleware" + }, { "label": "Structured Outputs", "to": "guides/structured-outputs" diff --git a/docs/guides/middleware.md b/docs/guides/middleware.md new file mode 100644 index 000000000..bf4753d26 --- /dev/null +++ b/docs/guides/middleware.md @@ -0,0 +1,652 @@ +--- +title: Middleware +id: middleware +order: 7 +--- + +Middleware lets you hook into every stage of the `chat()` lifecycle — from configuration to streaming, tool execution, usage tracking, and completion. You can observe, transform, or short-circuit behavior at each stage without modifying your adapter or tool implementations. + +Common use cases include: + +- **Logging and observability** — track token usage, tool execution timing, errors +- **Configuration transforms** — inject system prompts, adjust temperature per iteration, filter tools +- **Stream processing** — redact sensitive content, transform chunks, drop unwanted events +- **Tool call interception** — validate arguments, cache results, abort on dangerous calls +- **Side effects** — send analytics, update databases, trigger notifications + +## Quick Start + +Pass an array of middleware to the `chat()` function: + +```typescript +import { chat, type ChatMiddleware } from "@tanstack/ai"; +import { openaiText } from "@tanstack/ai-openai"; + +const logger: ChatMiddleware = { + name: "logger", + onStart: (ctx) => { + console.log(`[${ctx.requestId}] Chat started`); + }, + onFinish: (ctx, info) => { + console.log(`[${ctx.requestId}] Finished in ${info.duration}ms`); + }, +}; + +const stream = chat({ + adapter: openaiText("gpt-4o"), + messages: [{ role: "user", content: "Hello" }], + middleware: [logger], +}); +``` + +## Lifecycle Overview + +Every `chat()` invocation follows a predictable lifecycle. Middleware hooks fire at specific phases: + +```mermaid +graph TD + A["chat() called"] --> B["onConfig (phase: init)"] + B --> C[onStart] + C --> D["onConfig (phase: beforeModel)"] + D --> E["Adapter streams response"] + E --> F["onChunk (for each chunk)"] + F --> G{Tool calls?} + G -->|No| H[onUsage] + G -->|Yes| I[onBeforeToolCall] + I --> J[Tool executes] + J --> K[onAfterToolCall] + K --> L{Continue loop?} + L -->|Yes| D + L -->|No| H + H --> M{Outcome} + M -->|Success| N[onFinish] + M -->|Abort| O[onAbort] + M -->|Error| P[onError] + + style I fill:#e1f5ff + style J fill:#ffe1e1 + style N fill:#e1ffe1 + style O fill:#fff4e1 + style P fill:#ffe1e1 +``` + +### Phase Transitions + +The context's `phase` field tracks where you are in the lifecycle: + +| Phase | When | Hooks Called | +|-------|------|-------------| +| `init` | Once at startup | `onConfig` | +| `beforeModel` | Before each model call (per iteration) | `onConfig` | +| `modelStream` | While adapter streams chunks | `onChunk`, `onUsage` | +| `beforeTools` | Before tool execution | `onBeforeToolCall` | +| `afterTools` | After tool execution | `onAfterToolCall` | + +## Hooks Reference + +### onConfig + +Called twice per iteration: once during `init` (startup) and once during `beforeModel` (before each model call). Use it to transform the configuration that the model receives. + +Return a **partial** config object with only the fields you want to change — they are shallow-merged with the current config automatically. No need to spread the existing config. + +```typescript +const dynamicTemperature: ChatMiddleware = { + name: "dynamic-temperature", + onConfig: (ctx, config) => { + if (ctx.phase === "init") { + // Add a system prompt at startup — only systemPrompts is overwritten + return { + systemPrompts: [ + ...config.systemPrompts, + "You are a helpful assistant.", + ], + }; + } + + if (ctx.phase === "beforeModel" && ctx.iteration > 0) { + // Increase temperature on retries — other fields stay unchanged + return { + temperature: Math.min((config.temperature ?? 0.7) + 0.1, 1.0), + }; + } + }, +}; +``` + +**Config fields you can transform:** + +| Field | Type | Description | +|-------|------|-------------| +| `messages` | `ModelMessage[]` | Conversation history | +| `systemPrompts` | `string[]` | System prompts | +| `tools` | `Tool[]` | Available tools | +| `temperature` | `number` | Sampling temperature | +| `topP` | `number` | Nucleus sampling | +| `maxTokens` | `number` | Token limit | +| `metadata` | `Record` | Request metadata | +| `modelOptions` | `Record` | Provider-specific options | + +When multiple middleware define `onConfig`, the config is **piped** through them in order — each receives the merged config from the previous middleware. + +### onStart + +Called once after the initial `onConfig` completes. Use it for setup tasks like initializing timers or logging. + +```typescript +const timer: ChatMiddleware = { + name: "timer", + onStart: (ctx) => { + console.log(`Request ${ctx.requestId} started at iteration ${ctx.iteration}`); + }, +}; +``` + +### onChunk + +Called for every chunk streamed from the adapter. You can observe, transform, expand, or drop chunks. + +```typescript +const redactor: ChatMiddleware = { + name: "redactor", + onChunk: (ctx, chunk) => { + if (chunk.type === "TEXT_MESSAGE_CONTENT") { + // Transform: redact sensitive content + return { + ...chunk, + delta: chunk.delta.replace(/\b\d{3}-\d{2}-\d{4}\b/g, "[REDACTED]"), + }; + } + // Return void to pass through unchanged + }, +}; +``` + +**Return values:** + +| Return | Effect | +|--------|--------| +| `void` / `undefined` | Chunk passes through unchanged | +| `StreamChunk` | Replaces the original chunk | +| `StreamChunk[]` | Expands into multiple chunks | +| `null` | Drops the chunk entirely | + +When multiple middleware define `onChunk`, chunks flow through them in order. If one middleware drops a chunk (returns `null`), subsequent middleware never see it. + +### onBeforeToolCall + +Called before each tool executes. The first middleware that returns a non-void decision short-circuits — remaining middleware are skipped for that tool call. + +```typescript +const guard: ChatMiddleware = { + name: "guard", + onBeforeToolCall: (ctx, hookCtx) => { + // Block dangerous tools + if (hookCtx.toolName === "deleteDatabase") { + return { type: "abort", reason: "Dangerous operation blocked" }; + } + + // Validate and transform arguments + if (hookCtx.toolName === "search" && !hookCtx.args.limit) { + return { + type: "transformArgs", + args: { ...hookCtx.args, limit: 10 }, + }; + } + }, +}; +``` + +**Decision types:** + +| Decision | Effect | +|----------|--------| +| `void` / `undefined` | Continue normally, next middleware can decide | +| `{ type: 'transformArgs', args }` | Replace tool arguments before execution | +| `{ type: 'skip', result }` | Skip execution entirely, use provided result | +| `{ type: 'abort', reason? }` | Abort the entire chat run | + +The `hookCtx` provides: + +| Field | Type | Description | +|-------|------|-------------| +| `toolCall` | `ToolCall` | Raw tool call object | +| `tool` | `Tool \| undefined` | Resolved tool definition | +| `args` | `unknown` | Parsed arguments | +| `toolName` | `string` | Tool name | +| `toolCallId` | `string` | Tool call ID | + +### onAfterToolCall + +Called after each tool execution (or skip). All middleware run — there is no short-circuiting. + +```typescript +const toolLogger: ChatMiddleware = { + name: "tool-logger", + onAfterToolCall: (ctx, info) => { + if (info.ok) { + console.log(`${info.toolName} completed in ${info.duration}ms`); + } else { + console.error(`${info.toolName} failed:`, info.error); + } + }, +}; +``` + +The `info` object provides: + +| Field | Type | Description | +|-------|------|-------------| +| `toolCall` | `ToolCall` | Raw tool call object | +| `tool` | `Tool \| undefined` | Resolved tool definition | +| `toolName` | `string` | Tool name | +| `toolCallId` | `string` | Tool call ID | +| `ok` | `boolean` | Whether execution succeeded | +| `duration` | `number` | Execution time in milliseconds | +| `result` | `unknown` | Result (when `ok` is true) | +| `error` | `unknown` | Error (when `ok` is false) | + +### onUsage + +Called once per model iteration when the `RUN_FINISHED` chunk includes usage data. Receives the usage object directly. + +```typescript +const usageTracker: ChatMiddleware = { + name: "usage-tracker", + onUsage: (ctx, usage) => { + console.log( + `Iteration ${ctx.iteration}: ${usage.totalTokens} tokens` + ); + }, +}; +``` + +The `usage` object: + +| Field | Type | Description | +|-------|------|-------------| +| `promptTokens` | `number` | Input tokens | +| `completionTokens` | `number` | Output tokens | +| `totalTokens` | `number` | Total tokens | + +### Terminal Hooks: onFinish, onAbort, onError + +Exactly **one** terminal hook fires per `chat()` invocation. They are mutually exclusive: + +| Hook | When it fires | +|------|--------------| +| `onFinish` | Run completed normally | +| `onAbort` | Run was aborted (via `ctx.abort()`, an external `AbortSignal`, or a `{ type: 'abort' }` decision from `onBeforeToolCall`) | +| `onError` | An unhandled error occurred | + +```typescript +const terminal: ChatMiddleware = { + name: "terminal", + onFinish: (ctx, info) => { + console.log(`Finished: ${info.finishReason}, ${info.duration}ms`); + console.log(`Content: ${info.content}`); + if (info.usage) { + console.log(`Tokens: ${info.usage.totalTokens}`); + } + }, + onAbort: (ctx, info) => { + console.log(`Aborted: ${info.reason}, ${info.duration}ms`); + }, + onError: (ctx, info) => { + console.error(`Error after ${info.duration}ms:`, info.error); + }, +}; +``` + +## Context Object + +Every hook receives a `ChatMiddlewareContext` as its first argument. It provides request-scoped information and control functions: + +| Field | Type | Description | +|-------|------|-------------| +| `requestId` | `string` | Unique ID for this chat request | +| `streamId` | `string` | Unique ID for this stream | +| `conversationId` | `string \| undefined` | User-provided conversation ID | +| `phase` | `ChatMiddlewarePhase` | Current lifecycle phase | +| `iteration` | `number` | Agent loop iteration (0-indexed) | +| `chunkIndex` | `number` | Running count of chunks yielded | +| `signal` | `AbortSignal \| undefined` | External abort signal | +| `abort(reason?)` | `function` | Abort the run from within middleware | +| `context` | `unknown` | User-provided context value | +| `defer(promise)` | `function` | Register a non-blocking side-effect | + +### Aborting from Middleware + +Call `ctx.abort()` to gracefully stop the run. This triggers the `onAbort` terminal hook: + +```typescript +const timeout: ChatMiddleware = { + name: "timeout", + onChunk: (ctx) => { + if (ctx.chunkIndex > 1000) { + ctx.abort("Too many chunks"); + } + }, +}; +``` + +### Deferred Side Effects + +Use `ctx.defer()` to register promises that run after the terminal hook without blocking the stream: + +```typescript +const analytics: ChatMiddleware = { + name: "analytics", + onFinish: (ctx, info) => { + ctx.defer( + fetch("/api/analytics", { + method: "POST", + body: JSON.stringify({ + requestId: ctx.requestId, + duration: info.duration, + tokens: info.usage?.totalTokens, + }), + }) + ); + }, +}; +``` + +## Composing Multiple Middleware + +Middleware execute in array order. The ordering matters for hooks that pipe or short-circuit: + +```typescript +const stream = chat({ + adapter: openaiText("gpt-4o"), + messages, + middleware: [authMiddleware, loggingMiddleware, cachingMiddleware], +}); +``` + +### Composition Rules + +| Hook | Composition | Effect of Order | +|------|------------|----------------| +| `onConfig` | **Piped** — each receives previous output | Earlier middleware transforms first | +| `onStart` | Sequential | All run in order | +| `onChunk` | **Piped** — chunks flow through each middleware | If first drops a chunk, later middleware never see it | +| `onBeforeToolCall` | **First-win** — first non-void decision wins | Earlier middleware has priority | +| `onAfterToolCall` | Sequential | All run in order | +| `onUsage` | Sequential | All run in order | +| `onFinish/onAbort/onError` | Sequential | All run in order | + +## Built-in Middleware + +### toolCacheMiddleware + +Caches tool call results based on tool name and arguments. When a tool is called with the same name and arguments as a previous call, the cached result is returned immediately without re-executing the tool. + +```typescript +import { chat, toolCacheMiddleware } from "@tanstack/ai"; + +const stream = chat({ + adapter: openaiText("gpt-4o"), + messages, + tools: [weatherTool, stockTool], + middleware: [ + toolCacheMiddleware({ + ttl: 60_000, // Cache entries expire after 60 seconds + maxSize: 50, // Keep at most 50 entries (LRU eviction) + toolNames: ["getWeather"], // Only cache specific tools + }), + ], +}); +``` + +**Options:** + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `maxSize` | `number` | `100` | Maximum cache entries. Oldest evicted first (LRU). Only applies to the default in-memory storage. | +| `ttl` | `number` | `Infinity` | Time-to-live in milliseconds. Expired entries are not served. | +| `toolNames` | `string[]` | All tools | Only cache these tools. Others pass through. | +| `keyFn` | `(toolName, args) => string` | `JSON.stringify([toolName, args])` | Custom cache key derivation. | +| `storage` | `ToolCacheStorage` | In-memory Map | Custom storage backend. When provided, `maxSize` is ignored — the storage manages its own capacity. | + +**Behaviors:** + +- Only successful tool calls are cached — errors are never stored +- Cache hits trigger `{ type: 'skip', result }` via `onBeforeToolCall` +- LRU eviction: when `maxSize` is reached, the oldest entry is removed (default storage only) +- Cache hits refresh the entry's LRU position (moved to most-recently-used) + +**Custom key function** — useful when you want to ignore certain arguments: + +```typescript +toolCacheMiddleware({ + keyFn: (toolName, args) => { + // Ignore pagination, cache by query only + const { page, ...rest } = args as Record; + return JSON.stringify([toolName, rest]); + }, +}); +``` + +#### Custom Storage + +By default the cache lives in-memory and is scoped to a single `toolCacheMiddleware()` instance. Pass a `storage` option to use an external backend like Redis, localStorage, or a database. This also enables **sharing a cache across multiple `chat()` calls**. + +The storage interface: + +```typescript +import type { ToolCacheStorage, ToolCacheEntry } from "@tanstack/ai"; + +interface ToolCacheStorage { + getItem: (key: string) => ToolCacheEntry | undefined | Promise; + setItem: (key: string, value: ToolCacheEntry) => void | Promise; + deleteItem: (key: string) => void | Promise; +} + +// ToolCacheEntry is { result: unknown, timestamp: number } +``` + +All methods may return a `Promise` for async backends. The middleware handles TTL checking — your storage just needs to store and retrieve entries. + +**Redis example:** + +```typescript +import { createClient } from "redis"; +import { toolCacheMiddleware, type ToolCacheStorage } from "@tanstack/ai"; + +const redis = createClient(); + +const redisStorage: ToolCacheStorage = { + getItem: async (key) => { + const raw = await redis.get(`tool-cache:${key}`); + return raw ? JSON.parse(raw) : undefined; + }, + setItem: async (key, value) => { + await redis.set(`tool-cache:${key}`, JSON.stringify(value)); + }, + deleteItem: async (key) => { + await redis.del(`tool-cache:${key}`); + }, +}; + +const stream = chat({ + adapter, + messages, + tools: [weatherTool], + middleware: [toolCacheMiddleware({ storage: redisStorage, ttl: 60_000 })], +}); +``` + +**Sharing a cache across requests:** + +```typescript +// Create storage once, reuse across chat() calls +const sharedStorage: ToolCacheStorage = { + getItem: (key) => globalCache.get(key), + setItem: (key, value) => { globalCache.set(key, value); }, + deleteItem: (key) => { globalCache.delete(key); }, +}; + +// Both requests share the same cache +app.post("/api/chat", async (req) => { + const stream = chat({ + adapter, + messages: req.body.messages, + tools: [weatherTool], + middleware: [toolCacheMiddleware({ storage: sharedStorage })], + }); + return toServerSentEventsResponse(stream); +}); +``` + +## Recipes + +### Rate Limiting + +Limit the number of tool calls per request: + +```typescript +function rateLimitMiddleware(maxCalls: number): ChatMiddleware { + let toolCallCount = 0; + return { + name: "rate-limit", + onBeforeToolCall: (ctx, hookCtx) => { + toolCallCount++; + if (toolCallCount > maxCalls) { + return { + type: "abort", + reason: `Rate limit: exceeded ${maxCalls} tool calls`, + }; + } + }, + }; +} +``` + +### Audit Trail + +Log every action for compliance: + +```typescript +const auditTrail: ChatMiddleware = { + name: "audit-trail", + onStart: (ctx) => { + ctx.defer( + db.auditLog.create({ + requestId: ctx.requestId, + event: "chat_started", + timestamp: Date.now(), + }) + ); + }, + onAfterToolCall: (ctx, info) => { + ctx.defer( + db.auditLog.create({ + requestId: ctx.requestId, + event: "tool_executed", + toolName: info.toolName, + success: info.ok, + duration: info.duration, + timestamp: Date.now(), + }) + ); + }, + onFinish: (ctx, info) => { + ctx.defer( + db.auditLog.create({ + requestId: ctx.requestId, + event: "chat_finished", + duration: info.duration, + tokens: info.usage?.totalTokens, + timestamp: Date.now(), + }) + ); + }, +}; +``` + +### Per-Iteration Tool Swapping + +Expose different tools at different stages of the agent loop: + +```typescript +const toolSwapper: ChatMiddleware = { + name: "tool-swapper", + onConfig: (ctx, config) => { + if (ctx.phase !== "beforeModel") return; + + if (ctx.iteration === 0) { + // First iteration: only allow search + return { + tools: config.tools.filter((t) => t.name === "search"), + }; + } + // Later iterations: allow all tools + }, +}; +``` + +### Content Filtering + +Drop or transform chunks before they reach the consumer: + +```typescript +const contentFilter: ChatMiddleware = { + name: "content-filter", + onChunk: (ctx, chunk) => { + if (chunk.type === "TEXT_MESSAGE_CONTENT") { + if (containsProfanity(chunk.delta)) { + // Drop the chunk entirely + return null; + } + } + }, +}; +``` + +### Error Recovery with Retry Logging + +```typescript +const errorRecovery: ChatMiddleware = { + name: "error-recovery", + onError: (ctx, info) => { + ctx.defer( + alertService.send({ + level: "error", + message: `Chat ${ctx.requestId} failed after ${info.duration}ms`, + error: String(info.error), + }) + ); + }, +}; +``` + +## TypeScript Types + +All middleware types are exported from `@tanstack/ai`: + +```typescript +import type { + ChatMiddleware, + ChatMiddlewareContext, + ChatMiddlewarePhase, + ChatMiddlewareConfig, + ToolCallHookContext, + BeforeToolCallDecision, + AfterToolCallInfo, + UsageInfo, + FinishInfo, + AbortInfo, + ErrorInfo, + ToolCacheMiddlewareOptions, + ToolCacheStorage, + ToolCacheEntry, +} from "@tanstack/ai"; +``` + +## Next Steps + +- [Tools](./tools) — Learn about the isomorphic tool system +- [Agentic Cycle](./agentic-cycle) — Understand the multi-step agent loop +- [Observability](./observability) — Event-driven observability with the event client +- [Streaming](./streaming) — How streaming works in TanStack AI diff --git a/examples/ts-react-chat/src/components/Header.tsx b/examples/ts-react-chat/src/components/Header.tsx index 57745b7b0..7e5037a65 100644 --- a/examples/ts-react-chat/src/components/Header.tsx +++ b/examples/ts-react-chat/src/components/Header.tsx @@ -1,7 +1,7 @@ import { Link } from '@tanstack/react-router' import { useState } from 'react' -import { Guitar, Home, Menu, X } from 'lucide-react' +import { Guitar, Home, Image, Menu, X } from 'lucide-react' export default function Header() { const [isOpen, setIsOpen] = useState(false) @@ -57,6 +57,19 @@ export default function Header() { Home + setIsOpen(false)} + className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-2" + activeProps={{ + className: + 'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-2', + }} + > + + Image Gen + +
rootRouteImport, +} as any) const IndexRoute = IndexRouteImport.update({ id: '/', path: '/', @@ -24,6 +31,11 @@ const ApiTanchatRoute = ApiTanchatRouteImport.update({ path: '/api/tanchat', getParentRoute: () => rootRouteImport, } as any) +const ApiImageGenRoute = ApiImageGenRouteImport.update({ + id: '/api/image-gen', + path: '/api/image-gen', + getParentRoute: () => rootRouteImport, +} as any) const ExampleGuitarsIndexRoute = ExampleGuitarsIndexRouteImport.update({ id: '/example/guitars/', path: '/example/guitars/', @@ -37,12 +49,16 @@ const ExampleGuitarsGuitarIdRoute = ExampleGuitarsGuitarIdRouteImport.update({ export interface FileRoutesByFullPath { '/': typeof IndexRoute + '/image-gen': typeof ImageGenRoute + '/api/image-gen': typeof ApiImageGenRoute '/api/tanchat': typeof ApiTanchatRoute '/example/guitars/$guitarId': typeof ExampleGuitarsGuitarIdRoute '/example/guitars/': typeof ExampleGuitarsIndexRoute } export interface FileRoutesByTo { '/': typeof IndexRoute + '/image-gen': typeof ImageGenRoute + '/api/image-gen': typeof ApiImageGenRoute '/api/tanchat': typeof ApiTanchatRoute '/example/guitars/$guitarId': typeof ExampleGuitarsGuitarIdRoute '/example/guitars': typeof ExampleGuitarsIndexRoute @@ -50,6 +66,8 @@ export interface FileRoutesByTo { export interface FileRoutesById { __root__: typeof rootRouteImport '/': typeof IndexRoute + '/image-gen': typeof ImageGenRoute + '/api/image-gen': typeof ApiImageGenRoute '/api/tanchat': typeof ApiTanchatRoute '/example/guitars/$guitarId': typeof ExampleGuitarsGuitarIdRoute '/example/guitars/': typeof ExampleGuitarsIndexRoute @@ -58,14 +76,24 @@ export interface FileRouteTypes { fileRoutesByFullPath: FileRoutesByFullPath fullPaths: | '/' + | '/image-gen' + | '/api/image-gen' | '/api/tanchat' | '/example/guitars/$guitarId' | '/example/guitars/' fileRoutesByTo: FileRoutesByTo - to: '/' | '/api/tanchat' | '/example/guitars/$guitarId' | '/example/guitars' + to: + | '/' + | '/image-gen' + | '/api/image-gen' + | '/api/tanchat' + | '/example/guitars/$guitarId' + | '/example/guitars' id: | '__root__' | '/' + | '/image-gen' + | '/api/image-gen' | '/api/tanchat' | '/example/guitars/$guitarId' | '/example/guitars/' @@ -73,6 +101,8 @@ export interface FileRouteTypes { } export interface RootRouteChildren { IndexRoute: typeof IndexRoute + ImageGenRoute: typeof ImageGenRoute + ApiImageGenRoute: typeof ApiImageGenRoute ApiTanchatRoute: typeof ApiTanchatRoute ExampleGuitarsGuitarIdRoute: typeof ExampleGuitarsGuitarIdRoute ExampleGuitarsIndexRoute: typeof ExampleGuitarsIndexRoute @@ -80,6 +110,13 @@ export interface RootRouteChildren { declare module '@tanstack/react-router' { interface FileRoutesByPath { + '/image-gen': { + id: '/image-gen' + path: '/image-gen' + fullPath: '/image-gen' + preLoaderRoute: typeof ImageGenRouteImport + parentRoute: typeof rootRouteImport + } '/': { id: '/' path: '/' @@ -94,6 +131,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiTanchatRouteImport parentRoute: typeof rootRouteImport } + '/api/image-gen': { + id: '/api/image-gen' + path: '/api/image-gen' + fullPath: '/api/image-gen' + preLoaderRoute: typeof ApiImageGenRouteImport + parentRoute: typeof rootRouteImport + } '/example/guitars/': { id: '/example/guitars/' path: '/example/guitars' @@ -113,6 +157,8 @@ declare module '@tanstack/react-router' { const rootRouteChildren: RootRouteChildren = { IndexRoute: IndexRoute, + ImageGenRoute: ImageGenRoute, + ApiImageGenRoute: ApiImageGenRoute, ApiTanchatRoute: ApiTanchatRoute, ExampleGuitarsGuitarIdRoute: ExampleGuitarsGuitarIdRoute, ExampleGuitarsIndexRoute: ExampleGuitarsIndexRoute, diff --git a/examples/ts-react-chat/src/routes/api.image-gen.ts b/examples/ts-react-chat/src/routes/api.image-gen.ts new file mode 100644 index 000000000..d0b6d89f5 --- /dev/null +++ b/examples/ts-react-chat/src/routes/api.image-gen.ts @@ -0,0 +1,52 @@ +import { createFileRoute } from '@tanstack/react-router' +import { generateImage } from '@tanstack/ai' +import { openRouterImage } from '@tanstack/ai-openrouter' + +export const Route = createFileRoute('/api/image-gen')({ + server: { + handlers: { + POST: async ({ request }) => { + const body = await request.json() + const { prompt, model, size } = body as { + prompt: string + model: string + size?: string + } + + if (!prompt) { + return new Response(JSON.stringify({ error: 'Prompt is required' }), { + status: 400, + headers: { 'Content-Type': 'application/json' }, + }) + } + + try { + const result = await generateImage({ + adapter: openRouterImage( + (model || 'openai/gpt-5-image-mini') as 'openai/gpt-5-image-mini', + ), + prompt, + ...(size ? { size: size as any } : {}), + }) + + return new Response(JSON.stringify(result), { + headers: { 'Content-Type': 'application/json' }, + }) + } catch (error: any) { + console.error('[Image Gen API] Error:', { + message: error?.message, + name: error?.name, + status: error?.status, + stack: error?.stack, + }) + return new Response( + JSON.stringify({ + error: error.message || 'Image generation failed', + }), + { status: 500, headers: { 'Content-Type': 'application/json' } }, + ) + } + }, + }, + }, +}) diff --git a/examples/ts-react-chat/src/routes/api.tanchat.ts b/examples/ts-react-chat/src/routes/api.tanchat.ts index 62cb2f1e7..086bd00ee 100644 --- a/examples/ts-react-chat/src/routes/api.tanchat.ts +++ b/examples/ts-react-chat/src/routes/api.tanchat.ts @@ -11,8 +11,8 @@ import { anthropicText } from '@tanstack/ai-anthropic' import { geminiText } from '@tanstack/ai-gemini' import { openRouterText } from '@tanstack/ai-openrouter' import { grokText } from '@tanstack/ai-grok' +import type { AnyTextAdapter, ChatMiddleware } from '@tanstack/ai' import { groqText } from '@tanstack/ai-groq' -import type { AnyTextAdapter } from '@tanstack/ai' import { addToCartToolDef, addToWishListToolDef, @@ -71,6 +71,39 @@ const addToCartToolServer = addToCartToolDef.server((args, context) => { } }) +const loggingMiddleware: ChatMiddleware = { + name: 'logging', + onConfig(ctx, config) { + console.log( + `[logging] onConfig iteration=${ctx.iteration} model=${ctx.model} tools=${config.tools.length}`, + ) + }, + onStart(ctx) { + console.log(`[logging] onStart requestId=${ctx.requestId}`) + }, + onIteration(ctx, info) { + console.log(`[logging] onIteration iteration=${info.iteration}`) + }, + onBeforeToolCall(ctx, toolCtx) { + console.log(`[logging] onBeforeToolCall tool=${toolCtx.toolName}`) + }, + onAfterToolCall(ctx, info) { + console.log( + `[logging] onAfterToolCall tool=${info.toolName} result=${JSON.stringify(info.result).slice(0, 100)}`, + ) + }, + onFinish(ctx, info) { + console.log( + `[logging] onFinish reason=${info.finishReason} iterations=${ctx.iteration}`, + ) + }, + onUsage(ctx, usage) { + console.log( + `[logging] onUsage tokens=${usage.totalTokens} input=${usage.promptTokens} output=${usage.completionTokens}, total: ${usage.totalTokens}`, + ) + }, +} + export const Route = createFileRoute('/api/tanchat')({ server: { handlers: { @@ -168,6 +201,7 @@ export const Route = createFileRoute('/api/tanchat')({ addToWishListToolDef, getPersonalGuitarPreferenceToolDef, ], + middleware: [loggingMiddleware], systemPrompts: [SYSTEM_PROMPT], agentLoopStrategy: maxIterations(20), messages, diff --git a/examples/ts-react-chat/src/routes/image-gen.tsx b/examples/ts-react-chat/src/routes/image-gen.tsx new file mode 100644 index 000000000..739516662 --- /dev/null +++ b/examples/ts-react-chat/src/routes/image-gen.tsx @@ -0,0 +1,300 @@ +import { useState } from 'react' +import { createFileRoute } from '@tanstack/react-router' +import { Download, Loader2, Send, X } from 'lucide-react' + +interface GeneratedImage { + url?: string + b64Json?: string + revisedPrompt?: string +} + +interface ImageGenResult { + id: string + model: string + images: Array + usage?: { + inputTokens?: number + outputTokens?: number + totalTokens?: number + } +} + +const IMAGE_MODELS = [ + { value: 'openai/gpt-5-image-mini', label: 'OpenAI GPT-5 Image Mini' }, + { value: 'openai/gpt-5-image', label: 'OpenAI GPT-5 Image' }, + { + value: 'google/gemini-2.5-flash-image', + label: 'Gemini 2.5 Flash Image', + }, + { + value: 'google/gemini-2.5-flash-image-preview', + label: 'Gemini 2.5 Flash Image Preview', + }, + { + value: 'google/gemini-3-pro-image-preview', + label: 'Gemini 3 Pro Image Preview', + }, +] as const + +const IMAGE_SIZES = [ + { value: '1024x1024', label: '1024x1024 (1:1)' }, + { value: '1248x832', label: '1248x832 (3:2)' }, + { value: '832x1248', label: '832x1248 (2:3)' }, + { value: '1184x864', label: '1184x864 (4:3)' }, + { value: '864x1184', label: '864x1184 (3:4)' }, + { value: '1344x768', label: '1344x768 (16:9)' }, + { value: '768x1344', label: '768x1344 (9:16)' }, +] as const + +function ImageGenPage() { + const [prompt, setPrompt] = useState('') + const [model, setModel] = useState(IMAGE_MODELS[0].value) + const [size, setSize] = useState(IMAGE_SIZES[0].value) + const [isLoading, setIsLoading] = useState(false) + const [result, setResult] = useState(null) + const [error, setError] = useState(null) + const [history, setHistory] = useState< + Array<{ prompt: string; result: ImageGenResult }> + >([]) + + async function handleGenerate() { + if (!prompt.trim() || isLoading) return + + setIsLoading(true) + setError(null) + setResult(null) + + try { + const res = await fetch('/api/image-gen', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ prompt: prompt.trim(), model, size }), + }) + + if (!res.ok) { + const errBody = await res.json().catch(() => null) + throw new Error(errBody?.error || `Request failed (${res.status})`) + } + + const data: ImageGenResult = await res.json() + setResult(data) + setHistory((prev) => [{ prompt: prompt.trim(), result: data }, ...prev]) + } catch (err: any) { + setError(err.message || 'Something went wrong') + } finally { + setIsLoading(false) + } + } + + function getImageSrc(image: GeneratedImage): string | null { + if (image.url) return image.url + if (image.b64Json) return `data:image/png;base64,${image.b64Json}` + return null + } + + return ( +
+ {/* Controls bar */} +
+
+
+ + +
+
+ + +
+
+
+ + {/* Main content area */} +
+ {/* Current result */} + {result && result.images.length > 0 && ( +
+
+ Prompt:{' '} + {history[0]?.prompt} + {result.usage?.totalTokens != null && ( + + ({result.usage.totalTokens} tokens) + + )} +
+
+ {result.images.map((image, i) => { + const src = getImageSrc(image) + if (!src) return null + return ( +
+ {image.revisedPrompt + {image.revisedPrompt && ( +
+ Revised:{' '} + {image.revisedPrompt} +
+ )} + + + +
+ ) + })} +
+
+ )} + + {/* Loading */} + {isLoading && ( +
+ +

Generating image...

+
+ )} + + {/* Error */} + {error && ( +
+
+ +
+

Generation failed

+

{error}

+
+
+
+ )} + + {/* Empty state */} + {!result && !isLoading && !error && ( +
+
+ + + +
+

Enter a prompt below to generate an image

+
+ )} + + {/* History */} + {history.length > 1 && ( +
+

+ Previous Generations +

+
+ {history.slice(1).map((entry, i) => + entry.result.images.map((image, j) => { + const src = getImageSrc(image) + if (!src) return null + return ( +
+ {entry.prompt} +
+ {entry.prompt} +
+
+ ) + }), + )} +
+
+ )} +
+ + {/* Input area */} +
+
+
+
+