From fa5bc41d2df8a60c0a3e20405c1c2e395c214cf7 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 21 Feb 2026 17:23:40 +0000 Subject: [PATCH 1/3] fix core transform reliability and align docs Co-authored-by: David Wells --- README.md | 8 +-- docs/1-getting-started.md | 10 +-- docs/2-syntax-reference.md | 34 +++++----- docs/3-plugin-development.md | 48 +++++++------- docs/4-advanced-usage.md | 44 ++++++------- docs/contributing.md | 65 ++++++++----------- docs/troubleshooting.md | 52 +++++++-------- examples/generate-readme.js | 2 +- md.config.js | 2 +- packages/block-parser/src/index.js | 41 ++++++++++-- packages/block-replacer/src/index.js | 9 +-- packages/block-transformer/src/index.js | 28 ++++---- packages/core/_tests/transform-code.test.js | 31 +++++++++ packages/core/_tests/transform-remote.test.js | 25 +++++++ packages/core/package.json | 2 +- packages/core/src/cli-run.js | 39 +++++++++-- .../core/src/index.dependency-graph.test.js | 49 ++++++++++++++ packages/core/src/index.js | 52 +++++++++++---- packages/core/src/transforms/code/index.js | 32 +++++---- packages/core/src/utils/remoteRequest.js | 36 ++++++++-- update-syntax.js | 4 +- 21 files changed, 415 insertions(+), 198 deletions(-) create mode 100644 packages/core/src/index.dependency-graph.test.js diff --git a/README.md b/README.md index 74bec853..426bcdad 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ This `README.md` is generated with `markdown-magic` [view the raw file](https:// - + ## Install **Via npm** @@ -127,7 +127,7 @@ In NPM scripts, `npm run docs` would run the markdown magic and parse all the `. }, ``` -If you have a `markdown.config.js` file where `markdown-magic` is invoked, it will automatically use that as the configuration unless otherwise specified by `--config` flag. +If you have an `md.config.js` or `markdown.config.js` file where `markdown-magic` is invoked, it will automatically use that as the configuration unless otherwise specified by `--config` flag. ### Running programmatically @@ -191,7 +191,7 @@ jobs: - [markdown-autodocs](https://github.com/dineshsonachalam/markdown-autodocs) - Auto-generate docs from code - + ## Syntax Examples There are various syntax options. Choose your favorite. @@ -669,7 +669,7 @@ const config = { return '**⊂◉‿◉つ**' }, lolz() { - return `This section was generated by the cli config markdown.config.js file` + return `This section was generated by the cli config md.config.js file` }, /* Match */ pluginExample: require('./plugin-example')({ addNewLine: true }), diff --git a/docs/1-getting-started.md b/docs/1-getting-started.md index df256067..5e10e201 100644 --- a/docs/1-getting-started.md +++ b/docs/1-getting-started.md @@ -18,9 +18,9 @@ npm install markdown-magic --save-dev Add comment blocks to your markdown files to define where content should be generated: ```md - + This content will be dynamically replaced from the remote url - + ``` Then run markdown-magic to process your files. @@ -44,7 +44,7 @@ md-magic Process specific files with custom config: ```bash -md-magic --path '**/*.md' --config ./config.file.js +md-magic --files '**/*.md' --config ./config.file.js ``` ### NPM Scripts Integration @@ -54,7 +54,7 @@ Add to your `package.json`: ```json { "scripts": { - "docs": "md-magic --path '**/*.md'" + "docs": "md-magic --files '**/*.md'" } } ``` @@ -63,7 +63,7 @@ Run with: `npm run docs` ### Configuration File -Create a `markdown.config.js` file in your project root for automatic configuration: +Create an `md.config.js` or `markdown.config.js` file in your project root for automatic configuration: ```js module.exports = { diff --git a/docs/2-syntax-reference.md b/docs/2-syntax-reference.md index 92afaa89..f67240d6 100644 --- a/docs/2-syntax-reference.md +++ b/docs/2-syntax-reference.md @@ -14,11 +14,11 @@ All transform blocks follow this basic structure: ```md Content to be replaced - + ``` Where: -- `matchWord` is the opening keyword (default: `doc-gen`) +- `matchWord` is the opening keyword (default: `docs`) - `transformName` is the name of the transform to apply - `options` are optional parameters for the transform - Content between the tags will be replaced by the transform output @@ -30,9 +30,9 @@ Where: The simplest form - just specify the transform name and options: ```md - + content to be replaced - + ``` ### Curly Braces @@ -40,9 +40,9 @@ content to be replaced Wrap the transform name in curly braces: ```md - + content to be replaced - + ``` ### Square Brackets @@ -50,9 +50,9 @@ content to be replaced Wrap the transform name in square brackets: ```md - + content to be replaced - + ``` ### Parentheses @@ -60,9 +60,9 @@ content to be replaced Wrap the transform name in parentheses: ```md - + content to be replaced - + ``` ### Function Style @@ -70,34 +70,34 @@ content to be replaced Use function-like syntax with parentheses for options: ```md - content to be replaced - + ``` ## Option Formats ### String Options ```md - + ``` ### Boolean Options ```md - + ``` ### Array Options ```md - + ``` ### Multiple Options ```md - content to be replaced - + ``` ## Best Practices diff --git a/docs/3-plugin-development.md b/docs/3-plugin-development.md index c778f9b7..ac41c977 100644 --- a/docs/3-plugin-development.md +++ b/docs/3-plugin-development.md @@ -9,21 +9,23 @@ Learn how to create custom transforms (plugins) for markdown-magic to extend its ## Transform Basics -A transform is a function that takes content and options, then returns processed content: +A transform is a function that takes a single API object and returns processed content: ```js // Basic transform structure -function myTransform(content, options, config) { +function myTransform(api) { + const { content, options, settings } = api // Process the content return processedContent } ``` -### Parameters +### API Parameters - `content`: The content between the comment blocks - `options`: Parsed options from the comment block -- `config`: Global markdown-magic configuration +- `srcPath`: Current source markdown file path +- `settings`: Global markdown-magic configuration ## Creating Your First Transform @@ -31,7 +33,7 @@ function myTransform(content, options, config) { ```js // transforms/greeting.js -module.exports = function greeting(content, options) { +module.exports = function greeting({ options }) { const name = options.name || 'World' return `Hello, ${name}!` } @@ -39,8 +41,8 @@ module.exports = function greeting(content, options) { Usage: ```md - - + + ``` Result: `Hello, Alice!` @@ -52,7 +54,7 @@ Result: `Hello, Alice!` const fs = require('fs') const path = require('path') -module.exports = function include(content, options) { +module.exports = function include({ content, options }) { if (!options.src) { throw new Error('include transform requires "src" option') } @@ -70,8 +72,8 @@ module.exports = function include(content, options) { Usage: ```md - - + + ``` ## Async Transforms @@ -82,7 +84,7 @@ For operations that require network requests or file system operations: // transforms/fetchContent.js const fetch = require('node-fetch') -module.exports = async function fetchContent(content, options) { +module.exports = async function fetchContent({ content, options }) { if (!options.url) { throw new Error('fetchContent requires "url" option') } @@ -108,7 +110,7 @@ module.exports = async function fetchContent(content, options) { // transforms/template.js const mustache = require('mustache') -module.exports = function template(content, options) { +module.exports = function template({ content, options }) { const template = options.template || content const data = { ...options.data, @@ -126,7 +128,7 @@ module.exports = function template(content, options) { // transforms/codeStats.js const fs = require('fs') -module.exports = function codeStats(content, options) { +module.exports = function codeStats({ options }) { if (!options.src) { throw new Error('codeStats requires "src" option') } @@ -149,7 +151,7 @@ module.exports = function codeStats(content, options) { ```js // transforms/tableOfContents.js -module.exports = function tableOfContents(content, options) { +module.exports = function tableOfContents({ content, options }) { const format = options.format || 'markdown' const maxDepth = parseInt(options.maxDepth) || 3 const minDepth = parseInt(options.minDepth) || 1 @@ -194,7 +196,7 @@ function extractHeadings(content, minDepth, maxDepth) { ### In Configuration File ```js -// markdown.config.js +// md.config.js const greeting = require('./transforms/greeting') const include = require('./transforms/include') @@ -215,7 +217,7 @@ import markdownMagic from 'markdown-magic' const config = { transforms: { - myTransform: (content, options) => { + myTransform: ({ content, options }) => { return `Processed: ${content}` } } @@ -231,7 +233,7 @@ markdownMagic('./docs/*.md', config) Markdown-magic automatically parses options from the comment block: ```md - - + + `.trim()) }) diff --git a/docs/4-advanced-usage.md b/docs/4-advanced-usage.md index f8618f5d..36908c9e 100644 --- a/docs/4-advanced-usage.md +++ b/docs/4-advanced-usage.md @@ -14,14 +14,14 @@ This guide covers advanced patterns and techniques for using markdown-magic effe You can use multiple transform blocks in sequence to build complex documentation: ```md - - + + - - + + - - + + ``` ### Conditional Processing @@ -32,7 +32,7 @@ Use custom transforms to conditionally include content: // In your config file module.exports = { transforms: { - conditional: (content, options) => { + conditional: ({ content, options }) => { if (process.env.NODE_ENV === options.env) { return options.content || content } @@ -43,8 +43,8 @@ module.exports = { ``` ```md - - + + ``` ## Error Handling @@ -57,7 +57,7 @@ Handle errors gracefully in your transforms: // Custom transform with error handling module.exports = { transforms: { - safeRemote: async (content, options) => { + safeRemote: async ({ content, options }) => { try { const response = await fetch(options.url) return await response.text() @@ -85,7 +85,7 @@ const validateOptions = (options, required) => { module.exports = { transforms: { - strictRemote: (content, options) => { + strictRemote: ({ content, options }) => { validateOptions(options, ['url']) // ... rest of transform logic } @@ -104,7 +104,7 @@ const cache = new Map() module.exports = { transforms: { - cachedRemote: async (content, options) => { + cachedRemote: async ({ content, options }) => { const cacheKey = `remote:${options.url}` if (cache.has(cacheKey)) { @@ -142,16 +142,16 @@ Promise.all(promises).then(results => { ### Environment-Specific Configs ```js -// markdown.config.js +// md.config.js const isDev = process.env.NODE_ENV === 'development' module.exports = { - matchWord: 'doc-gen', + matchWord: 'docs', outputDir: isDev ? './docs-dev' : './docs', transforms: { // Environment-specific transforms ...(isDev && { - debug: (content, options) => { + debug: ({ content, options }) => { console.log('Debug transform:', options) return content } @@ -170,7 +170,7 @@ import markdownMagic from 'markdown-magic' // Config for documentation const docsConfig = { - matchWord: 'doc-gen', + matchWord: 'docs', transforms: { /* docs-specific transforms */ } } @@ -192,7 +192,7 @@ markdownMagic('./README.md', readmeConfig) ```js module.exports = { transforms: { - packageInfo: (content, options) => { + packageInfo: ({ content, options }) => { const pkg = require('./package.json') return ` @@ -220,7 +220,7 @@ const mustache = require('mustache') module.exports = { transforms: { - template: (content, options) => { + template: ({ content, options }) => { const template = options.template || content const data = options.data || {} @@ -232,9 +232,9 @@ module.exports = { Usage: ```md - + Hello {{name}}, you are a {{role}}! - + ``` ## Testing Custom Transforms @@ -249,7 +249,7 @@ describe('myTransform', () => { it('should process content correctly', () => { const content = 'original' const options = { param: 'value' } - const result = myTransform(content, options) + const result = myTransform({ content, options }) expect(result).toBe('expected output') }) @@ -297,7 +297,7 @@ const debug = require('debug')('markdown-magic:custom') module.exports = { transforms: { - debugTransform: (content, options) => { + debugTransform: ({ content, options }) => { debug('Processing with options:', options) // ... transform logic debug('Result:', result) diff --git a/docs/contributing.md b/docs/contributing.md index f929e956..625d695a 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -11,8 +11,8 @@ Thank you for your interest in contributing to markdown-magic! This guide will h ### Prerequisites -- Node.js (version 14 or higher) -- npm or yarn +- Node.js (version 18 or higher) +- pnpm (version 10 or higher) - Git ### Getting Started @@ -26,9 +26,7 @@ Thank you for your interest in contributing to markdown-magic! This guide will h 3. **Install dependencies**: ```bash - npm install - # or - yarn install + pnpm install ``` 4. **Create a feature branch**: @@ -41,12 +39,14 @@ Thank you for your interest in contributing to markdown-magic! This guide will h ``` markdown-magic/ ├── packages/ # Monorepo packages -│ ├── core/ # Core markdown-magic package -│ ├── cli/ # CLI package -│ └── transforms/ # Built-in transforms +│ ├── core/ # Core markdown-magic package +│ ├── block-parser/ # Comment block parser package +│ ├── block-replacer/ # Block replacement engine +│ ├── block-transformer/ # Transform orchestration engine +│ └── plugin-*/ # Optional plugin packages ├── examples/ # Usage examples ├── docs/ # Documentation -├── test/ # Test files +├── packages/**/_tests/ # Test files └── scripts/ # Build and development scripts ``` @@ -56,45 +56,31 @@ markdown-magic/ ```bash # Run all tests -npm test +pnpm test -# Run tests in watch mode -npm run test:watch - -# Run tests for specific package -npm run test:core +# Run tests for all packages directly +pnpm -r test ``` -### Linting and Formatting - -```bash -# Run linter -npm run lint - -# Auto-fix linting issues -npm run lint:fix - -# Format code -npm run format -``` +### Types and Build ### Building ```bash # Build all packages -npm run build +pnpm build -# Build specific package -npm run build:core +# Generate declaration files where configured +pnpm types ``` ### Testing Your Changes 1. **Link locally** to test in other projects: ```bash - npm link + pnpm link --global cd /path/to/test-project - npm link markdown-magic + pnpm link --global markdown-magic ``` 2. **Run examples**: @@ -105,8 +91,8 @@ npm run build:core 3. **Test CLI**: ```bash - npm run cli -- --help - npm run cli -- --path './test/*.md' + pnpm --filter markdown-magic run cli -- --help + pnpm --filter markdown-magic run cli -- --files './docs/*.md' ``` ## Types of Contributions @@ -186,7 +172,7 @@ New built-in transforms should: - **ES6+ features** are encouraged - **2 spaces** for indentation -- **Semicolons** are required +- **Semicolons** are generally omitted - **Single quotes** for strings - **Trailing commas** in multiline structures @@ -221,12 +207,13 @@ export default function mainFunction() {} ```js /** * Transform function description - * @param {string} content - The content to transform - * @param {object} options - Transform options - * @param {object} config - Global configuration + * @param {object} api - Transform API payload + * @param {string} api.content - The content to transform + * @param {object} api.options - Transform options + * @param {object} api.settings - Global configuration * @returns {string|Promise} Transformed content */ -function myTransform(content, options, config) { +function myTransform({ content, options, settings }) { // Implementation details } ``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b1fbfa44..63300fdb 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -56,8 +56,8 @@ npm update markdown-magic 2. **Typo in transform name**: ```md - ✅ - ❌ (case sensitive) + ✅ + ❌ (case sensitive) ``` 3. **Transform file not found**: @@ -75,9 +75,9 @@ npm update markdown-magic 1. **Check comment syntax**: ```md - ✅ - ❌ (extra dash) - ✅ + ❌ (extra dash) + ✅ - ❌ (wrong match word) + ❌ (wrong match word) ``` 3. **Check file glob pattern**: ```bash # Make sure your files match the pattern - md-magic --path '**/*.md' # All .md files - md-magic --path './docs/*.md' # Only docs folder + md-magic --files '**/*.md' # All .md files + md-magic --files './docs/*.md' # Only docs folder ``` ### Options Not Parsed @@ -108,24 +108,24 @@ npm update markdown-magic ```md - + - ❌ (missing quotes) - ❌ (spaces around =) - ❌ (unmatched quotes) + ❌ (missing quotes) + ❌ (spaces around =) + ❌ (unmatched quotes) ``` **Complex options**: ```md - + - + - @@ -143,7 +143,7 @@ npm update markdown-magic ```bash # Test your glob pattern ls **/*.md # Check if files exist - md-magic --path '**/*.md' # Use quotes for glob + md-magic --files '**/*.md' # Use quotes for glob ``` 2. **Check working directory**: @@ -209,14 +209,14 @@ chmod 755 docs/ **Solutions**: 1. **Use correct filename**: - - `markdown.config.js` (auto-detected) - - `md.config.js` (auto-detected) + - `md.config.js` (auto-detected, preferred) + - `markdown.config.js` (auto-detected, legacy) - Or specify: `--config ./my-config.js` 2. **Check file location**: ```bash # Config should be in project root or specify path - ls -la markdown.config.js + ls -la md.config.js md-magic --config ./path/to/config.js ``` @@ -420,8 +420,8 @@ module.exports = function myTransform(content, options) { 3. **Optimize file patterns**: ```bash # More specific patterns are faster - md-magic --path './docs/*.md' # ✅ Specific directory - md-magic --path '**/*.md' # ❌ Searches everywhere + md-magic --files './docs/*.md' # ✅ Specific directory + md-magic --files '**/*.md' # ❌ Searches everywhere ``` ## Getting Help @@ -443,13 +443,13 @@ node --version npm --version # Debug output -DEBUG=markdown-magic:* md-magic --path './problematic-file.md' 2> debug.log +DEBUG=markdown-magic:* md-magic --files './problematic-file.md' 2> debug.log # File contents (if not sensitive) cat problematic-file.md # Configuration -cat markdown.config.js +cat md.config.js ``` ### Minimal Reproduction @@ -461,15 +461,15 @@ Create a minimal example that reproduces the issue: const markdownMagic = require('markdown-magic') const content = ` - - + + ` require('fs').writeFileSync('test.md', content) markdownMagic('test.md', { transforms: { - problemTransform: (content, options) => { + problemTransform: ({ content, options }) => { // Minimal version of your problem return 'result' } diff --git a/examples/generate-readme.js b/examples/generate-readme.js index f6f3cb16..6895796e 100644 --- a/examples/generate-readme.js +++ b/examples/generate-readme.js @@ -95,7 +95,7 @@ const config = { return '**⊂◉‿◉つ**' }, lolz() { - return `This section was generated by the cli config markdown.config.js file` + return `This section was generated by the cli config md.config.js file` }, /* Match */ pluginExample: require('./plugin-example')({ addNewLine: true }), diff --git a/md.config.js b/md.config.js index d3eefa66..2dc35880 100644 --- a/md.config.js +++ b/md.config.js @@ -1,4 +1,4 @@ -/* CLI markdown.config.js file example */ +/* CLI md.config.js file example */ module.exports = { // handleOutputPath: (currentPath) => { // const newPath = 'x' + currentPath diff --git a/packages/block-parser/src/index.js b/packages/block-parser/src/index.js index 75cf88af..9a631734 100644 --- a/packages/block-parser/src/index.js +++ b/packages/block-parser/src/index.js @@ -173,6 +173,7 @@ const defaultOptions = { function parseBlocks(contents, opts = {}) { const _options = Object.assign({}, defaultOptions, opts) const { syntax, customPatterns, firstArgIsType } = _options + const getLineNumberAt = createLineNumberResolver(contents) // Extract regex source from open/close (handles RegExp objects and '/pattern/flags' strings) const openInfo = getRegexSource(opts.open !== undefined ? opts.open : _options.open) @@ -375,7 +376,7 @@ Details: const indent = spaces || '' const openStart = newMatches.index + indent.length const openEnd = openStart + fullComment.length - const lineNum = contents.substr(0, openStart).split('\n').length + const lineNum = getLineNumberAt(openStart) if (newMatches.index === newerRegex.lastIndex) { newerRegex.lastIndex++ @@ -439,8 +440,8 @@ Details: const openStart = newMatches.index + indent.length const openEnd = openStart + openTag.length const closeEnd = newerRegex.lastIndex - const lineOpen = contents.substr(0, openStart).split('\n').length - const lineClose = contents.substr(0, closeEnd).split('\n').length + const lineOpen = getLineNumberAt(openStart) + const lineClose = getLineNumberAt(closeEnd) const contentStart = openStart + openTag.length const contentEnd = contentStart + content.length @@ -533,8 +534,8 @@ Details: const closeEnd = newerRegex.lastIndex // const finIndentation = (lineOpen === lineClose) ? '' : indent - const lineOpen = contents.substr(0, openStart).split('\n').length - const lineClose = contents.substr(0, closeEnd).split('\n').length + const lineOpen = getLineNumberAt(openStart) + const lineClose = getLineNumberAt(closeEnd) const contentStart = openStart + openTag.length // + indent.length// - shift //+ indent.length const contentEnd = contentStart + content.length // + finIndentation.length // + shift @@ -634,6 +635,36 @@ function findLeadingIndent(str) { return (str.match(LEADING_INDENT_REGEX) || [])[1]?.length || 0 } +/** + * Build a fast line-number resolver for character offsets + * @param {string} input + * @returns {(position: number) => number} + */ +function createLineNumberResolver(input) { + const newlineIndexes = [] + for (let i = 0; i < input.length; i++) { + if (input.charCodeAt(i) === 10) { + newlineIndexes.push(i) + } + } + + return function getLineNumberAt(position) { + if (position <= 0) return 1 + + let low = 0 + let high = newlineIndexes.length + while (low < high) { + const mid = (low + high) >> 1 + if (newlineIndexes[mid] < position) { + low = mid + 1 + } else { + high = mid + } + } + return low + 1 + } +} + function verifyTagsBalanced(str, open, close) { const openCount = (str.match(open) || []).length // console.log('openCount', openCount) diff --git a/packages/block-replacer/src/index.js b/packages/block-replacer/src/index.js index 2d9e8c20..aab20536 100644 --- a/packages/block-replacer/src/index.js +++ b/packages/block-replacer/src/index.js @@ -16,6 +16,7 @@ const { blockTransformer } = require('comment-block-transformer') * @typedef {ProcessContentConfig & { * content?: string * srcPath?: string + * parsedBlocks?: import('comment-block-parser').ParseBlocksResult * outputPath?: string * dryRun?: boolean * patterns?: { @@ -82,11 +83,11 @@ async function processFile(opts = {}) { const outputDir = output.directory || opts.outputDir let srcPath = opts.srcPath - if (srcPath && content) { - throw new Error(`Can't set both "srcPath" & "content"`) - } let fileContents - if (content) { + if (typeof content === 'string' && srcPath) { + // Allow callers to provide preloaded content for srcPath files + fileContents = content + } else if (content) { const isFile = isValidFile(content) && content.indexOf('\n') === -1 srcPath = (isFile) ? content : undefined fileContents = (!isFile) ? content : undefined diff --git a/packages/block-transformer/src/index.js b/packages/block-transformer/src/index.js index b2542552..20762b36 100644 --- a/packages/block-transformer/src/index.js +++ b/packages/block-transformer/src/index.js @@ -71,6 +71,7 @@ const CLOSE_WORD = '/block' * @property {string} [srcPath] - The source path. * @property {string} [outputPath] - The output path. * @property {import('comment-block-parser').CustomPatterns} [customPatterns] - Custom regex patterns for open and close tags. + * @property {import('comment-block-parser').ParseBlocksResult} [parsedBlocks] - Optional pre-parsed block data to skip reparsing. */ /** @@ -112,18 +113,21 @@ async function blockTransformer(inputText, config) { // Don't default close - let undefined pass through to enable pattern mode in block-parser const close = opts.close !== undefined ? opts.close : (opts.open ? undefined : CLOSE_WORD) - let foundBlocks = {} - try { - foundBlocks = parseBlocks(inputText, { - syntax, - open, - close, - customPatterns, - firstArgIsType: true, - }) - } catch (e) { - const errMsg = (srcPath) ? `in ${srcPath}` : inputText - throw new Error(`${e.message}\nFix content in ${errMsg}\n`) + /** @type {ParseBlocksResult} */ + let foundBlocks = opts.parsedBlocks + if (!foundBlocks || !Array.isArray(foundBlocks.blocks)) { + try { + foundBlocks = parseBlocks(inputText, { + syntax, + open, + close, + customPatterns, + firstArgIsType: true, + }) + } catch (e) { + const errMsg = (srcPath) ? `in ${srcPath}` : inputText + throw new Error(`${e.message}\nFix content in ${errMsg}\n`) + } } diff --git a/packages/core/_tests/transform-code.test.js b/packages/core/_tests/transform-code.test.js index 201fb06b..6a0e15ca 100644 --- a/packages/core/_tests/transform-code.test.js +++ b/packages/core/_tests/transform-code.test.js @@ -6,6 +6,7 @@ const { test } = require('uvu') const assert = require('uvu/assert') const { markdownMagic } = require('../src') const { FIXTURE_DIR, MARKDOWN_FIXTURE_DIR, OUTPUT_DIR } = require('./config') +const TEMP_FIXTURE_DIR = path.join(FIXTURE_DIR, 'temp-code') function getNewFile(result) { if (!result.results || !result.results[0]) { @@ -16,6 +17,10 @@ function getNewFile(result) { const SILENT = true +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }) +} + test('CODE - local file inclusion', async () => { const fileName = 'transform-code.md' const filePath = path.join(MARKDOWN_FIXTURE_DIR, fileName) @@ -105,4 +110,30 @@ test('CODE - legacy colon syntax works', async () => { assert.ok(newContent.includes('module.exports.run'), 'legacy syntax processed correctly') }) +test('CODE - throws when id markers are missing', async () => { + const content = ` +original content +` + + ensureDir(TEMP_FIXTURE_DIR) + const tempFile = path.join(TEMP_FIXTURE_DIR, 'code-id-missing.md') + fs.writeFileSync(tempFile, content) + + let threw = false + try { + await markdownMagic(tempFile, { + open: 'docs', + close: '/docs', + outputDir: OUTPUT_DIR, + applyTransformsToSource: false, + silent: SILENT + }) + } catch (err) { + threw = true + assert.ok(err.message.includes('Missing MISSING_ID code section'), 'throws missing code section error') + } + + assert.ok(threw, 'should throw when CODE id markers are missing') +}) + test.run() diff --git a/packages/core/_tests/transform-remote.test.js b/packages/core/_tests/transform-remote.test.js index cc36f5d1..eeaad553 100644 --- a/packages/core/_tests/transform-remote.test.js +++ b/packages/core/_tests/transform-remote.test.js @@ -89,6 +89,31 @@ original content assert.ok(threw, 'should throw on invalid URL') }) +test('remote - throws on HTTP 404 response', async () => { + const content = ` +original content +` + + ensureDir(TEMP_FIXTURE_DIR) + const tempFile = path.join(TEMP_FIXTURE_DIR, 'remote-404.md') + fs.writeFileSync(tempFile, content) + + let threw = false + try { + await markdownMagic(tempFile, { + open: 'docs', + close: '/docs', + outputDir: OUTPUT_DIR, + applyTransformsToSource: false, + silent: SILENT + }) + } catch (err) { + threw = true + assert.ok(err.message.includes('status 404') || err.message.includes('404'), 'throws HTTP status error') + } + assert.ok(threw, 'should throw on 404 response') +}) + test('remote - fetches raw markdown from GitHub', async () => { const content = ` original diff --git a/packages/core/package.json b/packages/core/package.json index a902f99d..53833981 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -21,7 +21,7 @@ "types": "tsc --emitDeclarationOnly --outDir types", "docs": "node examples/generate-readme.js", "test": "uvu . '.test.([mc]js|[jt]sx?)$'", - "cli": "node ./cli.js --path 'README.md' --config ./markdown.config.js", + "cli": "node ./cli.js --files 'README.md' --config ./md.config.js", "bundle": "npm run bundle:all", "bundle:all": "npm run bundle:mac && npm run bundle:linux && npm run bundle:windows", "bundle:mac": "bun build ./cli.js --compile --minify --target=bun-darwin-arm64 --outfile dist/md-magic-darwin-arm64 && bun build ./cli.js --compile --minify --target=bun-darwin-x64 --outfile dist/md-magic-darwin-x64", diff --git a/packages/core/src/cli-run.js b/packages/core/src/cli-run.js index c233212a..9402628a 100644 --- a/packages/core/src/cli-run.js +++ b/packages/core/src/cli-run.js @@ -11,7 +11,7 @@ const { getGlobGroupsFromArgs } = require('./globparse') // const { uxParse } = require('./argparse/argparse') const argv = process.argv.slice(2) const cwd = process.cwd() -const defaultConfigPath = 'md.config.js' +const defaultConfigPaths = ['md.config.js', 'markdown.config.js'] /** * Render markdown with ANSI styling for terminal output @@ -100,6 +100,22 @@ async function getBaseDir(opts = {}) { return (gitDir) ? path.dirname(gitDir) : currentDir } +/** + * Find the first config file that exists in parent dirs + * @param {string} baseDir + * @param {Array} configPaths + * @returns {Promise} + */ +async function findFirstConfig(baseDir, configPaths = []) { + for (let i = 0; i < configPaths.length; i++) { + const configPath = configPaths[i] + const found = await findUp(baseDir, configPath) + if (found) { + return found + } + } +} + function findSingleDashStrings(arr) { return arr.filter(str => str.match(/^-[^-]/)) } @@ -110,8 +126,9 @@ async function runCli(options = {}, rawArgv) { Usage: md-magic [options] [files...] Options: - --files, --file Files or glob patterns to process - --config Path to config file (default: md.config.js) + --files, --file, --path + Files or glob patterns to process + --config Path to config file (default: md.config.js or markdown.config.js) --output Output directory --open Opening comment keyword (default: docs) --close Closing comment keyword (default: /docs) @@ -125,6 +142,7 @@ Options: Examples: md-magic README.md md-magic --files "**/*.md" + md-magic --path "**/*.md" # alias for --files md-magic --config ./my-config.js Stdin/stdout mode: @@ -210,14 +228,14 @@ Stdin/stdout mode: /** */ options.files = [] /* If raw args found, process them further */ - if (argv.length && (options._ && options._.length || (options.file || options.files))) { + if (argv.length && (options._ && options._.length || (options.file || options.files || options.path))) { // if (isGlob(argv[0])) { // console.log('glob', argv[0]) // options.glob = argv[0] // } const globParse = getGlobGroupsFromArgs(argv, { /* CLI args that should be glob keys */ - globKeys: ['files', 'file'] + globKeys: ['files', 'file', 'path'] }) const { globGroups, otherOpts } = globParse /* @@ -304,6 +322,10 @@ Stdin/stdout mode: if (globGroupByKey.files) { options.files = options.files.concat(globGroupByKey.files.values) } + if (globGroupByKey.path) { + options.files = options.files.concat(globGroupByKey.path.values) + delete options.path + } if (globGroupByKey['']) { options.files = options.files.concat(globGroupByKey[''].values) } @@ -327,6 +349,11 @@ Stdin/stdout mode: delete extraParse.files } + if (extraParse.path) { + options.files = options.files.concat(extraParse.path) + delete extraParse.path + } + if (extraParse['--files']) { options.files = options.files.concat(extraParse['--files']) delete extraParse['--files'] @@ -354,7 +381,7 @@ Stdin/stdout mode: configFile = opts.config } else { const baseDir = await getBaseDir() - configFile = await findUp(baseDir, defaultConfigPath) + configFile = await findFirstConfig(baseDir, defaultConfigPaths) } const config = (configFile) ? loadConfig(configFile) : {} const mergedConfig = { diff --git a/packages/core/src/index.dependency-graph.test.js b/packages/core/src/index.dependency-graph.test.js new file mode 100644 index 00000000..4043ea89 --- /dev/null +++ b/packages/core/src/index.dependency-graph.test.js @@ -0,0 +1,49 @@ +const { test } = require('uvu') +const assert = require('uvu/assert') +const { __private } = require('./') + +test('createDependencyGraph includes every known dependency edge', () => { + const blockItems = [ + { + id: '/docs/main.md', + blocks: [{}], + dependencies: ['/docs/dep-a.md', '/docs/dep-z.md'] + }, + { + id: '/docs/dep-a.md', + blocks: [{}], + dependencies: [] + }, + { + id: '/docs/dep-z.md', + blocks: [{}], + dependencies: [] + } + ] + + const { graph } = __private.createDependencyGraph(blockItems) + assert.ok(graph.find((edge) => edge[0] === '/docs/main.md' && edge[1] === '/docs/dep-a.md')) + assert.ok(graph.find((edge) => edge[0] === '/docs/main.md' && edge[1] === '/docs/dep-z.md')) + assert.is(graph.filter((edge) => edge[0] === '/docs/main.md').length, 2) +}) + +test('createDependencyGraph skips dependencies outside the file set', () => { + const blockItems = [ + { + id: '/docs/main.md', + blocks: [{}], + dependencies: ['/docs/dep-a.md', '/external/file.js'] + }, + { + id: '/docs/dep-a.md', + blocks: [{}], + dependencies: [] + } + ] + + const { graph } = __private.createDependencyGraph(blockItems) + assert.is(graph.length, 1) + assert.equal(graph[0], ['/docs/main.md', '/docs/dep-a.md']) +}) + +test.run() diff --git a/packages/core/src/index.js b/packages/core/src/index.js index d078a16d..83a5487b 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -344,22 +344,24 @@ async function markdownMagic(globOrOpts = {}, options = {}) { name: file, id: file, srcPath: file, - blocks: foundBlocks.blocks + blocks: foundBlocks.blocks, + parsedBlocks: foundBlocks } }) const blocks = blocksByPath.map((item) => { const dir = path.dirname(item.srcPath) - item.dependencies = [] + const dependencySet = new Set() item.blocks.forEach((block) => { if (block.options && block.options.src) { const resolvedPath = path.resolve(dir, block.options.src) // if (resolvedPath.match(/\.md$/)) { // console.log('resolvedPath', resolvedPath) - item.dependencies = item.dependencies.concat(resolvedPath) + dependencySet.add(resolvedPath) //} } }) + item.dependencies = Array.from(dependencySet) return item }) @@ -381,23 +383,23 @@ async function markdownMagic(globOrOpts = {}, options = {}) { /** */ // Convert items into a format suitable for toposort - const graph = blocks - .filter((item) => item.blocks && item.blocks.length) - .map((item) => { - return [ item.id, ...item.dependencies ] - }) + const { itemsWithBlocks, itemIds, graph } = createDependencyGraph(blocks) // console.log('graph', graph) // Perform the topological sort and reverse for execution order - const sortedIds = toposort(graph).reverse(); + const sortedIds = graph.length ? toposort.array(itemIds, graph).reverse() : itemIds // Reorder items based on sorted ids - const sortedItems = sortedIds.map(id => blocks.find(item => item.id === id)).filter(Boolean); + const itemById = new Map(itemsWithBlocks.map((item) => [item.id, item])) + const sortedItems = sortedIds.map((id) => itemById.get(id)).filter(Boolean) // topoSort(blocks) const orderedFiles = sortedItems.map((block) => block.id) // console.log('sortedItems', sortedItems) // console.log('orderedFiles', orderedFiles) + const fileContentByPath = new Map(files.map((file, i) => [file, fileContents[i]])) + const parsedBlocksByPath = new Map(blocksByPath.map((item) => [item.id, item.parsedBlocks])) + const processedFiles = [] await asyncForEach(orderedFiles, async (file) => { // logger('file', file) @@ -424,6 +426,8 @@ async function markdownMagic(globOrOpts = {}, options = {}) { // logger('newPath', newPath) const result = await processFile({ ...opts, + content: fileContentByPath.get(file), + parsedBlocks: parsedBlocksByPath.get(file), patterns, open, close, @@ -787,6 +791,29 @@ function changedFiles(files) { return files.filter(({ isChanged }) => isChanged) } +/** + * Create graph data for deterministic dependency ordering + * @param {Array<{id: string, blocks: Array, dependencies?: Array}>} blockItems + * @returns {{itemsWithBlocks: Array, itemIds: Array, graph: Array<[string, string]>}} + */ +function createDependencyGraph(blockItems = []) { + const itemsWithBlocks = blockItems.filter((item) => item.blocks && item.blocks.length) + const itemIds = itemsWithBlocks.map((item) => item.id) + const itemIdSet = new Set(itemIds) + const graph = itemsWithBlocks + .flatMap((item) => { + return (item.dependencies || []) + .filter((dependency) => itemIdSet.has(dependency)) + .map((dependency) => [item.id, dependency]) + }) + + return { + itemsWithBlocks, + itemIds, + graph + } +} + async function asyncForEach(array, callback) { for (let index = 0; index < array.length; index++) { await callback(array[index], index, array) @@ -802,5 +829,8 @@ module.exports = { parseMarkdown, blockTransformer, processFile, - stringUtils + stringUtils, + __private: { + createDependencyGraph + } } \ No newline at end of file diff --git a/packages/core/src/transforms/code/index.js b/packages/core/src/transforms/code/index.js index 022079b1..df468c4a 100644 --- a/packages/core/src/transforms/code/index.js +++ b/packages/core/src/transforms/code/index.js @@ -158,23 +158,29 @@ module.exports = async function CODE(api) { /* Check for Id */ if (id) { - const lines = code.split("\n") - const startLineIndex = lines.findIndex(line => line.includes(`CODE_SECTION:${id}:START`)); - const startLine = startLineIndex !== -1 ? startLineIndex : 0; + const lines = code.split('\n') + const startLineIndex = lines.findIndex((line) => line.includes(`CODE_SECTION:${id}:START`)) + const endLineIndex = lines.findIndex((line) => line.includes(`CODE_SECTION:${id}:END`)) + + if (startLineIndex === -1 || endLineIndex === -1) { + throw new Error(`Missing ${id} code section from ${codeFilePath}`) + } + + if (endLineIndex <= startLineIndex) { + throw new Error(`Invalid ${id} code section in ${codeFilePath}. End marker must be after start marker`) + } - const endLineIndex = lines.findIndex(line => line.includes(`CODE_SECTION:${id}:END`)); - const endLine = endLineIndex !== -1 ? endLineIndex : lines.length - 1; // console.log('startLine', startLine) // console.log('endLine', endLine) - if (startLine === -1 && endLine === -1) { - throw new Error(`Missing ${id} code section from ${codeFilePath}`) + const selectedLines = lines.slice(startLineIndex + 1, endLineIndex) + + if (!selectedLines.length) { + throw new Error(`Empty ${id} code section in ${codeFilePath}`) } - - const selectedLines = lines.slice(startLine + 1, endLine) - - const firstMatch = selectedLines[0] && selectedLines[0].match(/^(\s*)/); - const trimBy = firstMatch && firstMatch[1] ? firstMatch[1].length : 0; - const newValue = `${selectedLines.map(line => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, "")).join("\n")}` + + const firstMatch = selectedLines[0] && selectedLines[0].match(/^(\s*)/) + const trimBy = firstMatch && firstMatch[1] ? firstMatch[1].length : 0 + const newValue = `${selectedLines.map((line) => line.substring(trimBy).replace(/^\/\/ CODE_SECTION:INCLUDE /g, '')).join('\n')}` // console.log('newValue', newValue) code = newValue } diff --git a/packages/core/src/utils/remoteRequest.js b/packages/core/src/utils/remoteRequest.js index 67f6e107..549371ec 100644 --- a/packages/core/src/utils/remoteRequest.js +++ b/packages/core/src/utils/remoteRequest.js @@ -1,28 +1,52 @@ const fetch = require('node-fetch') function formatUrl(url = '') { - return url.match(/^https?:\/\//) ? url : `https://${url}` + if (typeof url !== 'string') return '' + const trimmed = url.trim() + if (!trimmed) return '' + return trimmed.match(/^https?:\/\//) ? trimmed : `https://${trimmed}` } async function remoteRequest(url, settings = {}, srcPath) { - let body const finalUrl = formatUrl(url) + const fixText = srcPath ? `\nFix "${url}" value in ${srcPath}` : '' + if (!finalUrl) { + const msg = `Invalid URL "${url}"${fixText}` + if (settings.failOnMissingRemote) { + throw new Error(msg) + } + console.log(msg) + return + } + // ignore demo url todo remove one day if (finalUrl === 'http://url-to-raw-md-file.md') { return } + + let response try { - const res = await fetch(finalUrl) - body = await res.text() + response = await fetch(finalUrl) } catch (e) { console.log(`⚠️ WARNING: REMOTE URL "${finalUrl}" NOT FOUND`) - const msg = (e.message || '').split('\n')[0] + `\nFix "${url}" value in ${srcPath}` + const msg = (e.message || '').split('\n')[0] + fixText console.log(msg) if (settings.failOnMissingRemote) { throw new Error(msg) } + return + } + + if (!response.ok) { + const msg = `Remote request failed with status ${response.status} (${response.statusText}) for "${finalUrl}"${fixText}` + console.log(`⚠️ WARNING: ${msg}`) + if (settings.failOnMissingRemote) { + throw new Error(msg) + } + return } - return body + + return response.text() } module.exports = { diff --git a/update-syntax.js b/update-syntax.js index 1e8b64cc..e90d8637 100644 --- a/update-syntax.js +++ b/update-syntax.js @@ -5,10 +5,10 @@ * to the new 'docs' syntax. It serves as an example of how to use the * generic migration utilities. * - * For creating custom migrations, see packages/block-migrate package + * For creating custom migrations, see packages/block-migrator package */ -const { migrateDocGenToDocs } = require('./packages/block-migrate/src/index'); +const { migrateDocGenToDocs } = require('./packages/block-migrator/src/index') async function updateMarkdownFiles() { console.log('Migrating doc-gen syntax to docs...\n'); From 0b8a16b69ffa3193a44bbc6fb077137395d0d078 Mon Sep 17 00:00:00 2001 From: David Wells Date: Sat, 21 Feb 2026 09:36:41 -0800 Subject: [PATCH 2/3] Update packages/block-transformer/src/index.js Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> --- packages/block-transformer/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/block-transformer/src/index.js b/packages/block-transformer/src/index.js index 20762b36..fdf72b16 100644 --- a/packages/block-transformer/src/index.js +++ b/packages/block-transformer/src/index.js @@ -113,7 +113,7 @@ async function blockTransformer(inputText, config) { // Don't default close - let undefined pass through to enable pattern mode in block-parser const close = opts.close !== undefined ? opts.close : (opts.open ? undefined : CLOSE_WORD) - /** @type {ParseBlocksResult} */ + /** @type {import('comment-block-parser').ParseBlocksResult} */ let foundBlocks = opts.parsedBlocks if (!foundBlocks || !Array.isArray(foundBlocks.blocks)) { try { From f1394c44a174101aac8e5972f0734c3011ca17fa Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Sat, 21 Feb 2026 17:50:20 +0000 Subject: [PATCH 3/3] fix(block-replacer): update test to reflect preloaded-content behavior When both srcPath and content are provided, content is now used as preloaded file contents instead of throwing an error. Update the test to match the new intentional behavior. Co-authored-by: David Wells --- packages/block-replacer/test/index.test.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/block-replacer/test/index.test.js b/packages/block-replacer/test/index.test.js index 65322a65..928f9e6b 100644 --- a/packages/block-replacer/test/index.test.js +++ b/packages/block-replacer/test/index.test.js @@ -192,20 +192,18 @@ test content assert.is(result.isChanged, false) }) -test('should handle both srcPath and content error', async () => { +test('should handle both srcPath and content using preloaded content', async () => { /** @type {ProcessFileOptions} */ const options = { - srcPath: '/some/path', - content: 'some content', + srcPath: '/some/nonexistent/path', + content: 'preloaded content', dryRun: true } - try { - await processFile(options) - assert.unreachable('Should have thrown an error') - } catch (error) { - assert.ok(error.message.includes('Can\'t set both "srcPath" & "content"')) - } + // When both srcPath and content are provided, content is used as preloaded + // file contents (caching optimization) — no error should be thrown. + const result = await processFile(options) + assert.ok(result, 'Should return a result without throwing') }) test('should handle file with output directory', async () => {