Generate structured llms.txt with progressive disclosure

docusaurus.config.js

@@ -379,13 +379,48 @@ module.exports = async function createConfigAsync() {
         require.resolve('./plugins/markdown-pages'),
         {
           docsDir: 'docs',
+          llmsTxt: {
+            siteUrl: 'https://docs.temporal.io',
+            title: 'Temporal Platform Documentation',
+            description: 'This file is a structured index of Temporal\'s documentation, following the llmstxt.org standard. Temporal is an open-source platform for building crash-proof applications that resume exactly where they left off after failures.',
+            rootContent:
+              'To fetch any page as raw Markdown, append `.md` to its URL path (e.g., `https://docs.temporal.io/workflows.md`).\n\n' +
+              'This documentation reflects the latest Temporal SDK and Platform behavior. If you\'re working with an older SDK version, verify API compatibility before applying suggestions from this content.',
+            excludePaths: ['tctl-v1'],
+            sections: [
+              {
+                title: 'Core Primitives',
+                description: 'Fundamental building blocks of the Temporal Platform. Read this section first for foundational concepts referenced everywhere else.',
+                inline: true,
+                autoDiscoverSubsections: 'encyclopedia',
+              },
+              {
+                title: 'Concepts',
+                description: 'What Temporal is and how it works.',
+                inline: true,
+                pages: [
+                  'temporal', 'glossary',
+                ],
+              },
+              { autoDiscover: 'develop', title: 'SDK Development Guides', description: 'Each SDK guide is organized by topic (e.g., workflows, activities, testing). All SDKs follow the same structure, with minor differences depending on language-specific features. Use these for language-specific implementation details.' },
+              { path: 'develop', title: 'Cross-SDK Development Guides', description: 'Development guidance that applies across SDKs (worker performance, safe deployments, plugins).' },
+              { path: 'cloud', title: 'Temporal Cloud', description: 'Deploy and manage Temporal Cloud' },
+              { path: 'evaluate', title: 'Evaluating Temporal', description: 'Background for deciding whether and how to adopt Temporal, including feature comparisons and use cases.' },
+              { path: 'production-deployment', title: 'Production Deployment', description: 'Deploy Temporal to production' },
+              { path: 'self-hosted-guide', title: 'Self-Hosted Service Guide', description: 'Deploy, configure, and operate a self-hosted Temporal Service.' },
+              { path: 'cli', title: 'CLI Reference', description: 'Temporal CLI command reference' },
+              { path: 'references', title: 'References', description: 'Configuration and API references' },
+              { path: 'troubleshooting', title: 'Troubleshooting', description: 'Common issues and solutions' },
+              { path: 'best-practices', title: 'Best Practices', description: 'Recommended patterns for Temporal' },
+            ],
+          },
         },
       ],
       [
         'docusaurus-plugin-llms',
         {
           // Generate both llms.txt (index) and llms-full.txt (complete content)
-          generateLLMsTxt: true,
+          generateLLMsTxt: false,
           generateLLMsFullTxt: true,
           generateMarkdownFiles: false,
 

plugins/markdown-pages/index.js

@@ -5,6 +5,7 @@ const matter = require('gray-matter');
 module.exports = function markdownPagesPlugin(context, options = {}) {
   const docsDir = path.resolve(context.siteDir, options.docsDir || 'docs');
   const routeBasePath = options.routeBasePath || '/';
+  const llmsTxt = options.llmsTxt || null;
 
   function walkDir(dir) {
     if (!fs.existsSync(dir)) return [];
@@ -30,6 +31,231 @@ module.exports = function markdownPagesPlugin(context, options = {}) {
     return `${dir}/${id}`;
   }
 
+  function findSection(page, sections) {
+    const urlPath = page.urlPath;
+    // Explicit page lists are intentional categorization and always take
+    // priority over automatic directory-based discovery.
+    for (const section of sections) {
+      if (section.pages && section.pages.some(p => urlPath === p)) {
+        return section;
+      }
+    }
+    for (const section of sections) {
+      if (section.autoDiscoverSubsections && page.relPath) {
+        const prefix = section.autoDiscoverSubsections + '/';
+        if (page.relPath.startsWith(prefix)) {
+          return section;
+        }
+      }
+    }
+    let bestMatch = null;
+    let bestLength = 0;
+    for (const section of sections) {
+      if (!section.path) continue;
+      const prefix = section.path;
+      if (
+        (urlPath === prefix || urlPath.startsWith(prefix + '/')) &&
+        prefix.length > bestLength
+      ) {
+        bestMatch = section;
+        bestLength = prefix.length;
+      }
+    }
+    return bestMatch;
+  }
+
+  function expandSections(sections, pages) {
+    const expanded = [];
+    for (const section of sections) {
+      if (section.autoDiscover) {
+        const prefix = section.autoDiscover;
+        const subdirs = new Set();
+        for (const page of pages) {
+          if (page.urlPath.startsWith(prefix + '/')) {
+            const rest = page.urlPath.slice(prefix.length + 1);
+            const subdir = rest.split('/')[0];
+            if (rest.includes('/')) {
+              subdirs.add(subdir);
+            }
+          }
+        }
+        const sorted = [...subdirs].sort();
+        for (const subdir of sorted) {
+          const sdkPath = `${prefix}/${subdir}`;
+          const indexPage = pages.find(
+            p => p.urlPath === sdkPath || p.urlPath === sdkPath + '/index'
+          );
+          const sdkTitle = indexPage ? indexPage.title : subdir;
+          expanded.push({
+            path: sdkPath,
+            title: sdkTitle,
+            _autoDiscovered: true,
+            _groupTitle: section.title,
+            _groupDescription: section.description || '',
+          });
+        }
+      } else {
+        expanded.push(section);
+      }
+    }
+    return expanded;
+  }
+
+  function generateLlmsTxtFiles(outDir, pages) {
+    const { siteUrl, title, description, rootContent, excludePaths = [] } = llmsTxt;
+    const sections = expandSections(llmsTxt.sections, pages);
+    const baseUrl = siteUrl.replace(/\/+$/, '');
+
+    function sectionKey(section) {
+      return section.path || section.title.toLowerCase().replace(/[^a-z0-9]+/g, '-');
+    }
+
+    const sectionBuckets = new Map();
+    for (const section of sections) {
+      sectionBuckets.set(sectionKey(section), []);
+    }
+    const otherPages = [];
+
+    for (const page of pages) {
+      if (excludePaths.some(p => page.urlPath === p || page.urlPath.startsWith(p + '/'))) {
+        continue;
+      }
+      const section = findSection(page, sections);
+      if (section) {
+        sectionBuckets.get(sectionKey(section)).push(page);
+      } else {
+        otherPages.push(page);
+      }
+    }
+
+    for (const section of sections) {
+      if (section.inline) continue;
+      const key = sectionKey(section);
+      const bucket = sectionBuckets.get(key);
+      bucket.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+      const lines = [`# ${section.title}`, ''];
+      if (section.description) {
+        lines.push(`> ${section.description}`, '');
+      }
+      for (const page of bucket) {
+        lines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`);
+      }
+      lines.push('');
+      const outputPath = path.join(outDir, key, 'llms.txt');
+      fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+      fs.writeFileSync(outputPath, lines.join('\n'));
+    }
+
+    const rootLines = [`# ${title}`, '', `> ${description}`, ''];
+    if (rootContent) {
+      rootLines.push(rootContent, '');
+    }
+
+    const inlineSections = sections.filter(s => s.inline);
+    const linkedSections = sections.filter(s => !s.inline);
+
+    for (const section of inlineSections) {
+      const key = sectionKey(section);
+      const bucket = sectionBuckets.get(key);
+      rootLines.push(`## ${section.title}`, '');
+      if (section.description) {
+        rootLines.push(section.description, '');
+      }
+      if (section.autoDiscoverSubsections) {
+        const prefix = section.autoDiscoverSubsections + '/';
+        const subGroups = new Map();
+        for (const page of bucket) {
+          if (!page.relPath || !page.relPath.startsWith(prefix)) continue;
+          const rest = page.relPath.slice(prefix.length);
+          const parts = rest.split('/');
+          if (parts.length < 2) continue;
+          const subdir = parts[0];
+          if (!subGroups.has(subdir)) subGroups.set(subdir, []);
+          subGroups.get(subdir).push(page);
+        }
+        const sortedDirs = [...subGroups.keys()].sort();
+        for (const subdir of sortedDirs) {
+          const subPages = subGroups.get(subdir);
+          const normalized = subdir.replace(/-/g, '');
+          const indexPage =
+            subPages.find(p => {
+              const filename = p.relPath.split('/').pop().replace(/\.(md|mdx)$/i, '');
+              return filename === 'index' || filename === subdir;
+            }) ||
+            subPages.find(p => {
+              const filename = p.relPath.split('/').pop().replace(/\.(md|mdx)$/i, '');
+              return filename === normalized || filename.endsWith('-overview');
+            });
+          const subTitle = indexPage
+            ? (indexPage.sidebarLabel || indexPage.title)
+            : subdir;
+          subPages.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+          rootLines.push(`### ${subTitle}`, '');
+          for (const page of subPages) {
+            rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`);
+          }
+          rootLines.push('');
+        }
+        const ungrouped = bucket.filter(p => {
+          if (!p.relPath || !p.relPath.startsWith(prefix)) return true;
+          const rest = p.relPath.slice(prefix.length);
+          return rest.split('/').length < 2;
+        });
+        if (ungrouped.length > 0) {
+          ungrouped.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+          rootLines.push(`### Other`, '');
+          for (const page of ungrouped) {
+            rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`);
+          }
+          rootLines.push('');
+        }
+      } else {
+        bucket.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+        for (const page of bucket) {
+          rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`);
+        }
+        rootLines.push('');
+      }
+    }
+
+    if (linkedSections.length > 0) {
+      let currentGroup = null;
+      for (const section of linkedSections) {
+        const group = section._groupTitle || 'Sections';
+        if (group !== currentGroup) {
+          if (currentGroup !== null) rootLines.push('');
+          rootLines.push(`## ${group}`, '');
+          if (section._groupDescription) {
+            rootLines.push(section._groupDescription, '');
+          }
+          currentGroup = group;
+        }
+        const desc = section.description ? `: ${section.description}` : '';
+        rootLines.push(
+          `- [${section.title}](${baseUrl}/${sectionKey(section)}/llms.txt)${desc}`
+        );
+      }
+      rootLines.push('');
+    }
+
+    otherPages.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+    if (otherPages.length > 0) {
+      rootLines.push('## Other', '');
+      for (const page of otherPages) {
+        rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`);
+      }
+      rootLines.push('');
+    }
+
+    const rootOutputPath = path.join(outDir, 'llms.txt');
+    fs.writeFileSync(rootOutputPath, rootLines.join('\n'));
+
+    const rootSize = Buffer.byteLength(rootLines.join('\n'), 'utf8');
+    console.log(
+      `[markdown-pages] Generated root llms.txt (${rootSize} bytes) and ${sections.length} section llms.txt files`
+    );
+  }
+
   return {
     name: 'markdown-pages',
 
@@ -46,6 +272,7 @@ module.exports = function markdownPagesPlugin(context, options = {}) {
       let generated = 0;
       let excluded = 0;
       let totalWarnings = 0;
+      const pages = [];
 
       for (const filePath of files) {
         const raw = fs.readFileSync(filePath, 'utf8');
@@ -69,13 +296,27 @@ module.exports = function markdownPagesPlugin(context, options = {}) {
           fs.writeFileSync(outputPath, markdown + '\n');
           totalWarnings += warnings.length;
           generated++;
+
+          if (llmsTxt && frontmatter.title) {
+            pages.push({
+              urlPath,
+              title: frontmatter.title,
+              sidebarLabel: frontmatter.sidebar_label || '',
+              description: frontmatter.description || '',
+              relPath: path.relative(docsDir, filePath).replace(/\\/g, '/'),
+            });
+          }
         }
       }
 
       console.log(
         `[markdown-pages] Generated ${generated} clean markdown files, ${excluded} excluded` +
           (totalWarnings ? `, ${totalWarnings} transform warnings` : '')
       );
+
+      if (llmsTxt) {
+        generateLlmsTxtFiles(outDir, pages);
+      }
     },
   };
 };

vercel.json

@@ -12,6 +12,15 @@
           "value": "noindex, nofollow"
         }
       ]
+    },
+    {
+      "source": "/((?!_next/static|_next/image|assets|img|scripts|favicon\\.ico).*)",
+      "headers": [
+        {
+          "key": "Link",
+          "value": "</llms.txt>; rel=\"llms-txt\""
+        }
+      ]
     }
   ],
   "redirects": [