feat(llms): implement LLMS.txt & markdown for llm consumption

dev/src/routes/about.mdx

@@ -1,5 +1,7 @@
 ---
 title: About
+llms:
+  exclude: true
 ---
 
 # About Page

dev/src/routes/dave.mdx

@@ -1,5 +1,6 @@
 ---
 title: What are we missing?
+description: This is a test page
 ---
 
 Title as defined by frontmatter: {frontmatter.title}

package.json

@@ -9,6 +9,8 @@
     "check": "biome check",
     "build": "tsc -b --force && node scripts/build.js",
     "format": "biome check --write #&& prettier . --write",
+    "test": "vitest run",
+    "test:watch": "vitest",
     "typecheck": "tsc -b",
     "publish:next": "pnpm i && pnpm build && pnpm publish --tag next",
     "publish:release": "pnpm i && pnpm build && pnpm publish",
@@ -109,7 +111,8 @@
     "tsx": "^4.19.3",
     "vfile": "^6.0.3",
     "vinxi": "^0.5.3",
-    "vite": "^6.1.1"
+    "vite": "^6.1.1",
+    "vitest": "^4.1.0"
   },
   "peerDependencies": {
     "@solidjs/start": "^1.0.8",
@@ -163,6 +166,9 @@
     "remark-directive": "^3.0.0",
     "remark-frontmatter": "^5.0.0",
     "remark-gfm": "^4.0.0",
+    "remark-mdx": "^3.1.1",
+    "remark-parse": "^11.0.0",
+    "remark-stringify": "^11.0.0",
     "solid-mdx": "^0.0.7",
     "source-map": "^0.7.6",
     "toml": "^3.0.0",

src/client/page-data.ts

@@ -12,13 +12,15 @@ export interface BaseFrontmatter {
 	title?: string;
 	titleTemplate?: string;
 	description?: string;
+	llms?: false | { exclude?: boolean };
 }
 
 interface CurrentPageData {
 	frontmatter: BaseFrontmatter;
 	toc?: Array<TableOfContentsItemData>;
 	editLink?: string;
 	lastUpdated?: number;
+	llmText?: string;
 }
 
 const [CurrentPageDataProvider, useCurrentPageDataContext] =

src/config/document-markdown.ts

@@ -0,0 +1,136 @@
+import remarkMdx from "remark-mdx";
+import remarkParse from "remark-parse";
+import remarkStringify from "remark-stringify";
+import { unified } from "unified";
+import { SKIP, visit } from "unist-util-visit";
+import { VFile } from "vfile";
+
+import type { RemarkPipelineConfig } from "./mdx.js";
+import { getRemarkPlugins } from "./mdx.js";
+import {
+	evaluateInlineFrontmatterExpression,
+	isInlineFrontmatterExpression,
+	remarkInlineFrontmatter,
+} from "./remark-plugins/inline-frontmatter.js";
+
+type DocumentMarkdownOptions = {
+	config?: RemarkPipelineConfig;
+	filePath?: string;
+};
+
+function renderExpressionValue(value: unknown): string | null {
+	if (value === null || value === undefined || typeof value === "boolean") {
+		return "";
+	}
+
+	if (
+		typeof value === "string" ||
+		typeof value === "number" ||
+		typeof value === "bigint"
+	) {
+		return String(value);
+	}
+
+	if (Array.isArray(value)) {
+		return value
+			.map((entry) => renderExpressionValue(entry))
+			.filter((entry): entry is string => entry !== null)
+			.join("");
+	}
+
+	return null;
+}
+
+function remarkDocumentInlineFrontmatter() {
+	return (tree: any) => {
+		visit(tree, (node, index, parent) => {
+			if (
+				(node.type !== "mdxTextExpression" &&
+					node.type !== "mdxFlowExpression") ||
+				index === undefined ||
+				parent === undefined ||
+				!isInlineFrontmatterExpression(node.value)
+			) {
+				return;
+			}
+
+			let value: unknown;
+			try {
+				value = evaluateInlineFrontmatterExpression(
+					tree.data.frontmatter,
+					node.value,
+				);
+			} catch {
+				return;
+			}
+
+			const text = renderExpressionValue(value);
+			if (text === null) return;
+
+			if (node.type === "mdxTextExpression") {
+				parent.children.splice(index, 1, {
+					type: "text",
+					value: text,
+				});
+				return [SKIP, index];
+			}
+
+			if (text === "") {
+				parent.children.splice(index, 1);
+				return [SKIP, index];
+			}
+
+			parent.children.splice(index, 1, {
+				type: "paragraph",
+				children: [{ type: "text", value: text }],
+			});
+			return [SKIP, index];
+		});
+	};
+}
+
+function remarkStripDocumentMetadata() {
+	return (tree: any) => {
+		visit(tree, (node, index, parent) => {
+			if (
+				index === undefined ||
+				parent === undefined ||
+				(node.type !== "yaml" &&
+					node.type !== "toml" &&
+					node.type !== "mdxjsEsm")
+			) {
+				return;
+			}
+
+			parent.children.splice(index, 1);
+			return [SKIP, index];
+		});
+	};
+}
+
+function getDocumentRemarkPlugins(config: RemarkPipelineConfig = {}) {
+	return getRemarkPlugins(config).map((plugin) =>
+		plugin === remarkInlineFrontmatter
+			? remarkDocumentInlineFrontmatter
+			: plugin,
+	);
+}
+
+export async function toDocumentMarkdown(
+	source: string,
+	options: DocumentMarkdownOptions = {},
+) {
+	const processor = unified()
+		.use(remarkParse)
+		.use(remarkMdx)
+		.use(getDocumentRemarkPlugins(options.config))
+		.use(remarkStripDocumentMetadata)
+		.use(remarkStringify)
+		.use(remarkMdx);
+
+	const file = await processor.process(
+		new VFile({ path: options.filePath, value: source }),
+	);
+
+	return String(file).trim();
+}

src/config/index.ts

@@ -17,6 +17,7 @@ export interface SolidBaseConfig<ThemeConfig> {
 	title?: string;
 	titleTemplate?: string;
 	description?: string;
+	llms?: boolean;
 	logo?: string;
 	issueAutolink?: IssueAutoLinkConfig | false;
 	lang?: string;
@@ -35,6 +36,7 @@ export interface SolidBaseConfig<ThemeConfig> {
 type ResolvedConfigKeys =
 	| "title"
 	| "description"
+	| "llms"
 	| "lang"
 	| "issueAutolink"
 	| "lastUpdated";
@@ -88,6 +90,7 @@ export function createWithSolidBase<ThemeConfig>(
 			title: "SolidBase",
 			description:
 				"Fully featured, fully customisable static site generation for SolidStart",
+			llms: false,
 			lang: "en-US",
 			issueAutolink: false,
 			lastUpdated: { dateStyle: "short", timeStyle: "short" },