feat: automate fetching and sanitization of webpack ecosystem readmes

.gitignore

@@ -3,3 +3,5 @@ out
 *.generated.*
 /.cache
 /pages/api
+/pages/loaders/
+/pages/plugins/

components/Footer/index.jsx

@@ -2,7 +2,7 @@ import GitHubIcon from '@node-core/ui-components/Icons/Social/GitHub';
 import LinkedInIcon from '@node-core/ui-components/Icons/Social/LinkedIn';
 import DiscordIcon from '@node-core/ui-components/Icons/Social/Discord';
 import XIcon from '@node-core/ui-components/Icons/Social/X';
-import { footer } from '#theme/site' with { type: 'json' };
+import { footer } from '#theme/site';
 
 import Logo from '#theme/Logo';
 import styles from './index.module.css';

components/NavBar.jsx

@@ -5,7 +5,7 @@ import GitHubIcon from '@node-core/ui-components/Icons/Social/GitHub';
 
 import SearchBox from '@node-core/doc-kit/src/generators/web/ui/components/SearchBox';
 import { useTheme } from '@node-core/doc-kit/src/generators/web/ui/hooks/useTheme.mjs';
-import { navbar } from '#theme/site' with { type: 'json' };
+import { navbar } from '#theme/site';
 import Logo from '#theme/Logo';
 
 /**

components/SideBar.jsx

@@ -1,5 +1,5 @@
 import SideBar from '@node-core/ui-components/Containers/Sidebar';
-import { sidebar } from '#theme/local/site' with { type: 'json' };
+import { sidebar } from '#theme/local/site';
 
 /** @param {string} url */
 const redirect = url => (window.location.href = url);
@@ -8,15 +8,24 @@ const PrefetchLink = props => <a {...props} rel="prefetch" />;
 
 const pathnameFor = path => path.replace(/\/index$/, '') || '/';
 
+const groupsFor = path => {
+  const segment = path.split('/').filter(Boolean)[0];
+  const matched = sidebar.filter(g => g.groupName.toLowerCase() === segment);
+  return matched.length > 0 ? matched : sidebar;
+};
+
 /**
  * Sidebar component for MDX documentation with page navigation.
  */
-export default ({ metadata }) => (
-  <SideBar
-    pathname={pathnameFor(metadata.path)}
-    groups={sidebar}
-    onSelect={redirect}
-    as={PrefetchLink}
-    title="Navigation"
-  />
-);
+export default ({ metadata }) => {
+  const path = pathnameFor(metadata.path);
+  return (
+    <SideBar
+      pathname={path}
+      groups={groupsFor(path)}
+      onSelect={redirect}
+      as={PrefetchLink}
+      title="Navigation"
+    />
+  );
+};

package.json

@@ -2,8 +2,11 @@
   "scripts": {
     "prep": "node scripts/prepare/index.mjs",
     "build:md": "node scripts/markdown/index.mjs",
+    "build:md:loaders": "node scripts/fetch-readmes.mjs --loaders",
+    "build:md:plugins": "node scripts/fetch-readmes.mjs --plugins",
+    "build:md:readmes": "node scripts/fetch-readmes.mjs",
     "build:html": "node scripts/html/index.mjs",
-    "build": "npm run prep && npm run build:md && npm run build:html",
+    "build": "npm run prep && npm run build:md && npm run build:md:readmes && npm run build:html",
     "lint": "eslint .",
     "lint:fix": "eslint --fix .",
     "format": "prettier --write .",

pages/site.mjs

@@ -0,0 +1,7 @@
+import base from './site.json' with { type: 'json' };
+import loadersSite from './loaders/site.json' with { type: 'json' };
+import pluginsSite from './plugins/site.json' with { type: 'json' };
+
+export const { navbar, footer } = base;
+
+export const sidebar = [...loadersSite.sidebar, ...pluginsSite.sidebar];

scripts/fetch-readmes.mjs

@@ -0,0 +1,196 @@
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const { GH_TOKEN } = process.env;
+
+const BASE_HEADERS = {
+  ...(GH_TOKEN && { Authorization: `Bearer ${GH_TOKEN}` }),
+  'X-GitHub-Api-Version': '2022-11-28',
+};
+
+const parseNextLink = linkHeader => {
+  if (!linkHeader) return null;
+  const match = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
+  return match ? match[1] : null;
+};
+
+const discoverRepos = async () => {
+  const loaders = [];
+  const plugins = [];
+  let url =
+    'https://api.github.com/orgs/webpack/repos?per_page=100&type=public';
+
+  while (url) {
+    const res = await fetch(url, { headers: BASE_HEADERS });
+    if (!res.ok)
+      throw new Error(
+        `Failed to list org repos: ${res.status} ${res.statusText}`
+      );
+
+    const repos = await res.json();
+    for (const repo of repos) {
+      if (repo.archived) continue;
+      if (repo.name.endsWith('-loader')) {
+        loaders.push(repo.full_name);
+      } else if (repo.name.endsWith('-plugin')) {
+        plugins.push(repo.full_name);
+      }
+    }
+
+    url = parseNextLink(res.headers.get('link'));
+  }
+
+  return { loaders, plugins };
+};
+
+const stripLeadingDiv = content =>
+  content.replace(/^\s*<div[\s\S]*?<\/div>\n*/i, '');
+
+// Remove badge lines - lines consisting only of [![...][ref]][ref] or [![...](url)](url) links
+const stripBadges = content =>
+  content
+    .replace(
+      /^(\[!\[[^\]]*\](?:\[[^\]]*\]|\([^)]*\))\]\s*(?:\[[^\]]*\]|\([^)]*\))\s*)+$/gm,
+      ''
+    )
+    .replace(/\n{3,}/g, '\n\n');
+
+// TODO: remove this allowlist once Shiki silently skips unknown languages instead of build errors.
+const SUPPORTED_LANGS = new Set([
+  'bash',
+  'c',
+  'c++',
+  'cjs',
+  'coffee',
+  'coffeescript',
+  'console',
+  'cpp',
+  'diff',
+  'docker',
+  'dockerfile',
+  'glsl',
+  'gql',
+  'graphql',
+  'http',
+  'ini',
+  'java',
+  'javascript',
+  'js',
+  'json',
+  'jsx',
+  'mjs',
+  'powershell',
+  'ps',
+  'ps1',
+  'regex',
+  'regexp',
+  'sh',
+  'shell',
+  'shellscript',
+  'shellsession',
+  'sql',
+  'ts',
+  'tsx',
+  'typescript',
+  'xml',
+  'yaml',
+  'yml',
+  'zsh',
+]);
+
+const sanitizeCodeFences = content =>
+  content.replace(/^```([a-zA-Z0-9_+-]+)\b/gm, (match, lang) =>
+    SUPPORTED_LANGS.has(lang.toLowerCase()) ? match : '```'
+  );
+
+// remark-gfm does not support GitHub alert syntax (> [!TYPE]); rewrite to bold label inside the blockquote.
+const GFM_ALERT_LABELS = {
+  NOTE: 'Note',
+  TIP: 'Tip',
+  IMPORTANT: 'Important',
+  WARNING: 'Warning',
+  CAUTION: 'Caution',
+};
+const GFM_ALERT_RE =
+  /^([ \t]*>[ \t]*)\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\][ \t]*$/gim;
+
+const transformGfmAlerts = content =>
+  content.replace(
+    GFM_ALERT_RE,
+    (_, prefix, type) => `${prefix}**${GFM_ALERT_LABELS[type]}:**`
+  );
+
+const processContent = content =>
+  transformGfmAlerts(sanitizeCodeFences(stripBadges(stripLeadingDiv(content))));
+
+const fetchReadme = async fullName => {
+  const url = `https://raw.githubusercontent.com/${fullName}/HEAD/README.md`;
+  const res = await fetch(url);
+  return res.ok
+    ? { ok: true, text: await res.text() }
+    : { ok: false, status: res.status };
+};
+
+const processRepos = async (repos, { groupName, basePath, outputDir }) => {
+  mkdirSync(outputDir, { recursive: true });
+  const repoName = r => r.split('/')[1];
+  console.log(
+    `Discovered ${groupName.toLowerCase()}: ${repos.map(repoName).join(', ')}`
+  );
+
+  const fetched = [];
+  for (const fullName of repos) {
+    const name = repoName(fullName);
+    const result = await fetchReadme(fullName);
+    if (!result.ok) {
+      console.log(`Failed: ${name} — ${result.status}`);
+      continue;
+    }
+    const content = processContent(result.text);
+    writeFileSync(join(outputDir, `${name}.md`), content, 'utf8');
+    fetched.push(name);
+    console.log(`Fetched: ${name}`);
+  }
+
+  const siteJson = {
+    sidebar: [
+      {
+        groupName,
+        items: fetched
+          .sort()
+          .map(name => ({ link: `${basePath}/${name}`, label: name })),
+      },
+    ],
+  };
+  writeFileSync(
+    join(outputDir, 'site.json'),
+    JSON.stringify(siteJson, null, 2) + '\n',
+    'utf8'
+  );
+  console.log(
+    `Written: ${outputDir}/site.json (${fetched.length} ${groupName.toLowerCase()})`
+  );
+};
+
+const args = process.argv.slice(2);
+const runLoaders = args.includes('--loaders') || args.length === 0;
+const runPlugins = args.includes('--plugins') || args.length === 0;
+
+const root = join(import.meta.dirname, '..');
+const { loaders, plugins } = await discoverRepos();
+
+if (runLoaders) {
+  await processRepos(loaders, {
+    groupName: 'Loaders',
+    basePath: '/loaders',
+    outputDir: join(root, 'pages/loaders'),
+  });
+}
+
+if (runPlugins) {
+  await processRepos(plugins, {
+    groupName: 'Plugins',
+    basePath: '/plugins',
+    outputDir: join(root, 'pages/plugins'),
+  });
+}

scripts/html/doc-kit.config.mjs

@@ -40,10 +40,12 @@ export default {
     useAbsoluteURLs: true,
     remoteConfigUrl: null,
     imports: {
-      '#theme/local/site': join(ROOT, inputDir, 'site.json'),
+      '#theme/local/site': VERSION
+        ? join(inputDir, 'site.json')
+        : join(ROOT, 'pages/site.mjs'),
 
       '#theme/Sidebar': join(ROOT, 'components/SideBar.jsx'),
-      '#theme/site': join(ROOT, 'pages/site.json'),
+      '#theme/site': join(ROOT, 'pages/site.mjs'),
       '#theme/Layout': join(ROOT, 'components/Layout.jsx'),
       '#theme/Navigation': join(ROOT, 'components/NavBar.jsx'),
       '#theme/Footer': join(ROOT, 'components/Footer/index.jsx'),