diff --git a/docs/docs/ai-friendly-docs.md b/docs/docs/ai-friendly-docs.md new file mode 100644 index 000000000..358ebec33 --- /dev/null +++ b/docs/docs/ai-friendly-docs.md @@ -0,0 +1,66 @@ +--- +sidebar_position: 90 +sidebar_label: AI-Friendly Docs +title: AI-Friendly Documentation +--- + +# AI-Friendly Documentation + +Endive documentation is designed to be easily consumed by LLMs, AI agents, and developer tools. All machine-readable files are auto-generated at build time from the same source as the human-facing site, using the [llms.txt specification](https://llmstxt.org/). + +## Available Endpoints + +| Endpoint | Description | +|----------|-------------| +| [`/llms.txt`](https://endive.run/llms.txt) | Lightweight index of all docs and blog posts with one-line descriptions | +| [`/llms-full.txt`](https://endive.run/llms-full.txt) | Full documentation content concatenated into a single file | +| [`/sitemap.xml`](https://endive.run/sitemap.xml) | Standard sitemap for crawler discovery | +| [`/robots.txt`](https://endive.run/robots.txt) | Permits all crawlers including AI agents | + +Every documentation page also has a **raw Markdown** version available by appending `.md` to its path. For example: + +- HTML: [`/docs/core/host-functions`](https://endive.run/docs/core/host-functions) +- Markdown: [`/docs/core/host-functions.md`](https://endive.run/docs/core/host-functions.md) + +## Usage Examples + +### Feed Endive docs to an LLM in one shot + +Fetch `llms-full.txt` and include it in your prompt context: + +```bash +curl -s https://endive.run/llms-full.txt | wc -c +# ~86 KB — fits easily in any modern LLM context window +``` + +### Let an AI agent discover relevant pages + +An agent can fetch `llms.txt` first to find the right page, then fetch only that page's Markdown: + +```bash +# 1. Fetch the index +curl -s https://endive.run/llms.txt + +# 2. Fetch a specific page as clean Markdown +curl -s https://endive.run/docs/core/host-functions.md +``` + +### Configure AI coding tools + +Many AI-powered development tools support `llms.txt` natively. Point them to `https://endive.run/llms.txt` to give them access to the full Endive documentation. + +## What Gets Included + +- All **documentation pages** covering installation, core concepts, compilation, WASI, security, and more +- All **published blog posts** +- Content is cleaned: front matter and test harness comments are stripped, leaving only the documentation text + +These files are regenerated on every site build, so they always reflect the latest documentation. + + diff --git a/docs/package.json b/docs/package.json index cbd2e569b..b05a08b8a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -5,7 +5,7 @@ "scripts": { "docusaurus": "docusaurus", "start": "docusaurus start", - "build": "docusaurus build", + "build": "docusaurus build && node scripts/generate-llm-files.mjs", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/docs/scripts/generate-llm-files.mjs b/docs/scripts/generate-llm-files.mjs new file mode 100644 index 000000000..ceb639407 --- /dev/null +++ b/docs/scripts/generate-llm-files.mjs @@ -0,0 +1,257 @@ +import { readFileSync, writeFileSync, mkdirSync, readdirSync, statSync, existsSync } from 'fs'; +import { join, dirname, relative, basename, extname } from 'path'; + +const DOCS_DIR = join(import.meta.dirname, '..', 'docs'); +const BLOG_DIR = join(import.meta.dirname, '..', 'blog'); +const BUILD_DIR = join(import.meta.dirname, '..', 'build'); +const BASE_URL = 'https://endive.run'; +const PROJECT_TITLE = 'Endive'; +const PROJECT_DESC = 'Endive is a JVM native WebAssembly runtime with zero native dependencies, hosted by the Bytecode Alliance.'; + +function parseFrontMatter(content) { + const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); + if (!match) return { meta: {}, body: content }; + + const meta = {}; + for (const line of match[1].split('\n')) { + const kv = line.match(/^(\w[\w_-]*)\s*:\s*(.+)$/); + if (kv) { + let val = kv[2].trim(); + if (val === 'true') val = true; + else if (val === 'false') val = false; + else if (/^\d+$/.test(val)) val = parseInt(val, 10); + else val = val.replace(/^['"]|['"]$/g, ''); + meta[kv[1]] = val; + } + } + return { meta, body: match[2] }; +} + +function stripHtmlComments(content) { + return content.replace(//g, ''); +} + +function cleanContent(content) { + const { body } = parseFrontMatter(content); + let cleaned = stripHtmlComments(body); + cleaned = cleaned.replace(/\n{3,}/g, '\n\n'); + return cleaned.trim() + '\n'; +} + +function extractDescription(body) { + const lines = stripHtmlComments(body).split('\n'); + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) continue; + if (trimmed.startsWith('#')) continue; + if (trimmed.startsWith(':::')) continue; + if (trimmed.startsWith('![')) continue; + if (trimmed.startsWith('```')) continue; + let desc = trimmed.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1'); + if (desc.length > 120) desc = desc.slice(0, 117) + '...'; + return desc; + } + return ''; +} + +function findMarkdownFiles(dir) { + const results = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) { + results.push(...findMarkdownFiles(full)); + } else if (extname(entry) === '.md') { + results.push(full); + } + } + return results; +} + +function getCategoryPosition(filePath) { + const catFile = join(dirname(filePath), '_category_.yml'); + if (!existsSync(catFile)) return 0; + const content = readFileSync(catFile, 'utf8'); + const match = content.match(/position:\s*(\d+)/); + return match ? parseInt(match[1], 10) : 999; +} + +function getDocEntries() { + const files = findMarkdownFiles(DOCS_DIR); + const entries = []; + + for (const file of files) { + const content = readFileSync(file, 'utf8'); + const { meta, body } = parseFrontMatter(content); + const relPath = relative(DOCS_DIR, file); + + let urlPath; + if (relPath === 'index.md') { + urlPath = '/docs/'; + } else if (basename(file) === 'index.md') { + urlPath = '/docs/' + dirname(relPath) + '/'; + } else { + urlPath = '/docs/' + relPath.replace(/\.md$/, ''); + } + + entries.push({ + file, + relPath, + urlPath, + title: meta.title || meta.sidebar_label || basename(file, '.md'), + sidebarPosition: meta.sidebar_position || 999, + categoryPosition: relPath === 'index.md' ? -1 : getCategoryPosition(file), + description: extractDescription(body), + content, + }); + } + + entries.sort((a, b) => { + if (a.categoryPosition !== b.categoryPosition) return a.categoryPosition - b.categoryPosition; + return a.sidebarPosition - b.sidebarPosition; + }); + + return entries; +} + +function getBlogEntries() { + if (!existsSync(BLOG_DIR)) return []; + const files = readdirSync(BLOG_DIR) + .filter(f => extname(f) === '.md') + .map(f => join(BLOG_DIR, f)) + .sort() + .reverse(); + + const entries = []; + for (const file of files) { + const content = readFileSync(file, 'utf8'); + const { meta, body } = parseFrontMatter(content); + if (meta.unlisted) continue; + + const slug = meta.slug || basename(file, '.md').replace(/^\d{4}-\d{2}-\d{2}-/, ''); + entries.push({ + file, + urlPath: '/blog/' + slug, + title: meta.title || slug, + description: extractDescription(body), + content, + }); + } + return entries; +} + +function generateLlmsTxt(docEntries, blogEntries) { + const lines = [ + `# ${PROJECT_TITLE}`, + '', + `> ${PROJECT_DESC}`, + '', + '## Docs', + '', + ]; + + for (const entry of docEntries) { + lines.push(`- [${entry.title}](${BASE_URL}${entry.urlPath}): ${entry.description}`); + } + + if (blogEntries.length > 0) { + lines.push('', '## Blog', ''); + for (const entry of blogEntries) { + lines.push(`- [${entry.title}](${BASE_URL}${entry.urlPath}): ${entry.description}`); + } + } + + lines.push('', '## Optional', ''); + lines.push(`- [llms-full.txt](${BASE_URL}/llms-full.txt): Full documentation content in a single file`); + lines.push(''); + + return lines.join('\n'); +} + +function generateLlmsFullTxt(docEntries, blogEntries) { + const sections = [ + `# ${PROJECT_TITLE}`, + '', + `> ${PROJECT_DESC}`, + '', + ]; + + for (const entry of docEntries) { + sections.push('---', ''); + sections.push(`## ${entry.title}`); + sections.push(''); + sections.push(cleanContent(entry.content)); + } + + for (const entry of blogEntries) { + sections.push('---', ''); + sections.push(`## Blog: ${entry.title}`); + sections.push(''); + sections.push(cleanContent(entry.content)); + } + + return sections.join('\n'); +} + +function copyCleanedMarkdown(docEntries) { + for (const entry of docEntries) { + const outPath = join(BUILD_DIR, 'docs', entry.relPath); + mkdirSync(dirname(outPath), { recursive: true }); + writeFileSync(outPath, cleanContent(entry.content), 'utf8'); + } +} + +function docUrlToHtmlPath(entry) { + const rel = entry.relPath; + if (rel === 'index.md') { + return join(BUILD_DIR, 'docs', 'index.html'); + } + if (basename(rel) === 'index.md') { + return join(BUILD_DIR, 'docs', dirname(rel), 'index.html'); + } + return join(BUILD_DIR, 'docs', rel.replace(/\.md$/, ''), 'index.html'); +} + +function injectAlternateLinks(docEntries) { + let injected = 0; + for (const entry of docEntries) { + const htmlPath = docUrlToHtmlPath(entry); + if (!existsSync(htmlPath)) { + console.warn(` warn: HTML not found: ${htmlPath}`); + continue; + } + + const mdUrl = '/docs/' + entry.relPath; + const linkTag = ``; + let html = readFileSync(htmlPath, 'utf8'); + + if (html.includes('rel="alternate" type="text/markdown"')) continue; + + html = html.replace('', `${linkTag}`); + writeFileSync(htmlPath, html, 'utf8'); + injected++; + } + return injected; +} + +console.log('Generating LLM-friendly files...'); + +const docEntries = getDocEntries(); +const blogEntries = getBlogEntries(); + +console.log(` Found ${docEntries.length} doc pages, ${blogEntries.length} published blog posts`); + +const llmsTxt = generateLlmsTxt(docEntries, blogEntries); +writeFileSync(join(BUILD_DIR, 'llms.txt'), llmsTxt, 'utf8'); +console.log(` Generated llms.txt (${llmsTxt.length} bytes)`); + +const llmsFullTxt = generateLlmsFullTxt(docEntries, blogEntries); +writeFileSync(join(BUILD_DIR, 'llms-full.txt'), llmsFullTxt, 'utf8'); +console.log(` Generated llms-full.txt (${llmsFullTxt.length} bytes)`); + +copyCleanedMarkdown(docEntries); +console.log(` Copied ${docEntries.length} cleaned .md files`); + +const injected = injectAlternateLinks(docEntries); +console.log(` Injected ${injected} tags`); + +console.log('Done.'); diff --git a/docs/static/robots.txt b/docs/static/robots.txt new file mode 100644 index 000000000..31ba5473d --- /dev/null +++ b/docs/static/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://endive.run/sitemap.xml diff --git a/docs/tests/approvals/docs-ai-friendly-docs.md.approved.txt b/docs/tests/approvals/docs-ai-friendly-docs.md.approved.txt new file mode 100644 index 000000000..c6cac6926 --- /dev/null +++ b/docs/tests/approvals/docs-ai-friendly-docs.md.approved.txt @@ -0,0 +1 @@ +empty