diff --git a/docusaurus.config.js b/docusaurus.config.js index ae5a372f38..ab150b0995 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -379,13 +379,48 @@ module.exports = async function createConfigAsync() { require.resolve('./plugins/markdown-pages'), { docsDir: 'docs', + llmsTxt: { + siteUrl: 'https://docs.temporal.io', + title: 'Temporal Platform Documentation', + description: 'This file is a structured index of Temporal\'s documentation, following the llmstxt.org standard. Temporal is an open-source platform for building crash-proof applications that resume exactly where they left off after failures.', + rootContent: + 'To fetch any page as raw Markdown, append `.md` to its URL path (e.g., `https://docs.temporal.io/workflows.md`).\n\n' + + 'This documentation reflects the latest Temporal SDK and Platform behavior. If you\'re working with an older SDK version, verify API compatibility before applying suggestions from this content.', + excludePaths: ['tctl-v1'], + sections: [ + { + title: 'Core Primitives', + description: 'Fundamental building blocks of the Temporal Platform. Read this section first for foundational concepts referenced everywhere else.', + inline: true, + autoDiscoverSubsections: 'encyclopedia', + }, + { + title: 'Concepts', + description: 'What Temporal is and how it works.', + inline: true, + pages: [ + 'temporal', 'glossary', + ], + }, + { autoDiscover: 'develop', title: 'SDK Development Guides', description: 'Each SDK guide is organized by topic (e.g., workflows, activities, testing). All SDKs follow the same structure, with minor differences depending on language-specific features. Use these for language-specific implementation details.' }, + { path: 'develop', title: 'Cross-SDK Development Guides', description: 'Development guidance that applies across SDKs (worker performance, safe deployments, plugins).' }, + { path: 'cloud', title: 'Temporal Cloud', description: 'Deploy and manage Temporal Cloud' }, + { path: 'evaluate', title: 'Evaluating Temporal', description: 'Background for deciding whether and how to adopt Temporal, including feature comparisons and use cases.' }, + { path: 'production-deployment', title: 'Production Deployment', description: 'Deploy Temporal to production' }, + { path: 'self-hosted-guide', title: 'Self-Hosted Service Guide', description: 'Deploy, configure, and operate a self-hosted Temporal Service.' }, + { path: 'cli', title: 'CLI Reference', description: 'Temporal CLI command reference' }, + { path: 'references', title: 'References', description: 'Configuration and API references' }, + { path: 'troubleshooting', title: 'Troubleshooting', description: 'Common issues and solutions' }, + { path: 'best-practices', title: 'Best Practices', description: 'Recommended patterns for Temporal' }, + ], + }, }, ], [ 'docusaurus-plugin-llms', { // Generate both llms.txt (index) and llms-full.txt (complete content) - generateLLMsTxt: true, + generateLLMsTxt: false, generateLLMsFullTxt: true, generateMarkdownFiles: false, diff --git a/plugins/markdown-pages/index.js b/plugins/markdown-pages/index.js index 375c6ab62b..fd50116a55 100644 --- a/plugins/markdown-pages/index.js +++ b/plugins/markdown-pages/index.js @@ -5,6 +5,7 @@ const matter = require('gray-matter'); module.exports = function markdownPagesPlugin(context, options = {}) { const docsDir = path.resolve(context.siteDir, options.docsDir || 'docs'); const routeBasePath = options.routeBasePath || '/'; + const llmsTxt = options.llmsTxt || null; function walkDir(dir) { if (!fs.existsSync(dir)) return []; @@ -30,6 +31,231 @@ module.exports = function markdownPagesPlugin(context, options = {}) { return `${dir}/${id}`; } + function findSection(page, sections) { + const urlPath = page.urlPath; + // Explicit page lists are intentional categorization and always take + // priority over automatic directory-based discovery. + for (const section of sections) { + if (section.pages && section.pages.some(p => urlPath === p)) { + return section; + } + } + for (const section of sections) { + if (section.autoDiscoverSubsections && page.relPath) { + const prefix = section.autoDiscoverSubsections + '/'; + if (page.relPath.startsWith(prefix)) { + return section; + } + } + } + let bestMatch = null; + let bestLength = 0; + for (const section of sections) { + if (!section.path) continue; + const prefix = section.path; + if ( + (urlPath === prefix || urlPath.startsWith(prefix + '/')) && + prefix.length > bestLength + ) { + bestMatch = section; + bestLength = prefix.length; + } + } + return bestMatch; + } + + function expandSections(sections, pages) { + const expanded = []; + for (const section of sections) { + if (section.autoDiscover) { + const prefix = section.autoDiscover; + const subdirs = new Set(); + for (const page of pages) { + if (page.urlPath.startsWith(prefix + '/')) { + const rest = page.urlPath.slice(prefix.length + 1); + const subdir = rest.split('/')[0]; + if (rest.includes('/')) { + subdirs.add(subdir); + } + } + } + const sorted = [...subdirs].sort(); + for (const subdir of sorted) { + const sdkPath = `${prefix}/${subdir}`; + const indexPage = pages.find( + p => p.urlPath === sdkPath || p.urlPath === sdkPath + '/index' + ); + const sdkTitle = indexPage ? indexPage.title : subdir; + expanded.push({ + path: sdkPath, + title: sdkTitle, + _autoDiscovered: true, + _groupTitle: section.title, + _groupDescription: section.description || '', + }); + } + } else { + expanded.push(section); + } + } + return expanded; + } + + function generateLlmsTxtFiles(outDir, pages) { + const { siteUrl, title, description, rootContent, excludePaths = [] } = llmsTxt; + const sections = expandSections(llmsTxt.sections, pages); + const baseUrl = siteUrl.replace(/\/+$/, ''); + + function sectionKey(section) { + return section.path || section.title.toLowerCase().replace(/[^a-z0-9]+/g, '-'); + } + + const sectionBuckets = new Map(); + for (const section of sections) { + sectionBuckets.set(sectionKey(section), []); + } + const otherPages = []; + + for (const page of pages) { + if (excludePaths.some(p => page.urlPath === p || page.urlPath.startsWith(p + '/'))) { + continue; + } + const section = findSection(page, sections); + if (section) { + sectionBuckets.get(sectionKey(section)).push(page); + } else { + otherPages.push(page); + } + } + + for (const section of sections) { + if (section.inline) continue; + const key = sectionKey(section); + const bucket = sectionBuckets.get(key); + bucket.sort((a, b) => a.urlPath.localeCompare(b.urlPath)); + const lines = [`# ${section.title}`, '']; + if (section.description) { + lines.push(`> ${section.description}`, ''); + } + for (const page of bucket) { + lines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`); + } + lines.push(''); + const outputPath = path.join(outDir, key, 'llms.txt'); + fs.mkdirSync(path.dirname(outputPath), { recursive: true }); + fs.writeFileSync(outputPath, lines.join('\n')); + } + + const rootLines = [`# ${title}`, '', `> ${description}`, '']; + if (rootContent) { + rootLines.push(rootContent, ''); + } + + const inlineSections = sections.filter(s => s.inline); + const linkedSections = sections.filter(s => !s.inline); + + for (const section of inlineSections) { + const key = sectionKey(section); + const bucket = sectionBuckets.get(key); + rootLines.push(`## ${section.title}`, ''); + if (section.description) { + rootLines.push(section.description, ''); + } + if (section.autoDiscoverSubsections) { + const prefix = section.autoDiscoverSubsections + '/'; + const subGroups = new Map(); + for (const page of bucket) { + if (!page.relPath || !page.relPath.startsWith(prefix)) continue; + const rest = page.relPath.slice(prefix.length); + const parts = rest.split('/'); + if (parts.length < 2) continue; + const subdir = parts[0]; + if (!subGroups.has(subdir)) subGroups.set(subdir, []); + subGroups.get(subdir).push(page); + } + const sortedDirs = [...subGroups.keys()].sort(); + for (const subdir of sortedDirs) { + const subPages = subGroups.get(subdir); + const normalized = subdir.replace(/-/g, ''); + const indexPage = + subPages.find(p => { + const filename = p.relPath.split('/').pop().replace(/\.(md|mdx)$/i, ''); + return filename === 'index' || filename === subdir; + }) || + subPages.find(p => { + const filename = p.relPath.split('/').pop().replace(/\.(md|mdx)$/i, ''); + return filename === normalized || filename.endsWith('-overview'); + }); + const subTitle = indexPage + ? (indexPage.sidebarLabel || indexPage.title) + : subdir; + subPages.sort((a, b) => a.urlPath.localeCompare(b.urlPath)); + rootLines.push(`### ${subTitle}`, ''); + for (const page of subPages) { + rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`); + } + rootLines.push(''); + } + const ungrouped = bucket.filter(p => { + if (!p.relPath || !p.relPath.startsWith(prefix)) return true; + const rest = p.relPath.slice(prefix.length); + return rest.split('/').length < 2; + }); + if (ungrouped.length > 0) { + ungrouped.sort((a, b) => a.urlPath.localeCompare(b.urlPath)); + rootLines.push(`### Other`, ''); + for (const page of ungrouped) { + rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`); + } + rootLines.push(''); + } + } else { + bucket.sort((a, b) => a.urlPath.localeCompare(b.urlPath)); + for (const page of bucket) { + rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`); + } + rootLines.push(''); + } + } + + if (linkedSections.length > 0) { + let currentGroup = null; + for (const section of linkedSections) { + const group = section._groupTitle || 'Sections'; + if (group !== currentGroup) { + if (currentGroup !== null) rootLines.push(''); + rootLines.push(`## ${group}`, ''); + if (section._groupDescription) { + rootLines.push(section._groupDescription, ''); + } + currentGroup = group; + } + const desc = section.description ? `: ${section.description}` : ''; + rootLines.push( + `- [${section.title}](${baseUrl}/${sectionKey(section)}/llms.txt)${desc}` + ); + } + rootLines.push(''); + } + + otherPages.sort((a, b) => a.urlPath.localeCompare(b.urlPath)); + if (otherPages.length > 0) { + rootLines.push('## Other', ''); + for (const page of otherPages) { + rootLines.push(`- [${page.title}](${baseUrl}/${page.urlPath}.md)`); + } + rootLines.push(''); + } + + const rootOutputPath = path.join(outDir, 'llms.txt'); + fs.writeFileSync(rootOutputPath, rootLines.join('\n')); + + const rootSize = Buffer.byteLength(rootLines.join('\n'), 'utf8'); + console.log( + `[markdown-pages] Generated root llms.txt (${rootSize} bytes) and ${sections.length} section llms.txt files` + ); + } + return { name: 'markdown-pages', @@ -46,6 +272,7 @@ module.exports = function markdownPagesPlugin(context, options = {}) { let generated = 0; let excluded = 0; let totalWarnings = 0; + const pages = []; for (const filePath of files) { const raw = fs.readFileSync(filePath, 'utf8'); @@ -69,6 +296,16 @@ module.exports = function markdownPagesPlugin(context, options = {}) { fs.writeFileSync(outputPath, markdown + '\n'); totalWarnings += warnings.length; generated++; + + if (llmsTxt && frontmatter.title) { + pages.push({ + urlPath, + title: frontmatter.title, + sidebarLabel: frontmatter.sidebar_label || '', + description: frontmatter.description || '', + relPath: path.relative(docsDir, filePath).replace(/\\/g, '/'), + }); + } } } @@ -76,6 +313,10 @@ module.exports = function markdownPagesPlugin(context, options = {}) { `[markdown-pages] Generated ${generated} clean markdown files, ${excluded} excluded` + (totalWarnings ? `, ${totalWarnings} transform warnings` : '') ); + + if (llmsTxt) { + generateLlmsTxtFiles(outDir, pages); + } }, }; }; diff --git a/vercel.json b/vercel.json index 504b5b8ccc..fcbbb27c13 100644 --- a/vercel.json +++ b/vercel.json @@ -12,6 +12,15 @@ "value": "noindex, nofollow" } ] + }, + { + "source": "/((?!_next/static|_next/image|assets|img|scripts|favicon\\.ico).*)", + "headers": [ + { + "key": "Link", + "value": "; rel=\"llms-txt\"" + } + ] } ], "redirects": [