✨ NEW: Add glob key (#6)

Store toctree list items as one of:
`GlobItem`, `RefItem`, `UrlItem`,
add glob logic to `append_toctrees`,
add addtional file creation to `create_site_from_toc`.

Author: chrisjsewell

README.md

@@ -38,7 +38,12 @@ The value of the `doc` key will be a path to a file (relative to the `conf.py`)
 Each document can only occur once in the ToC!
 ```
 
-Documents can then have a `parts` key - denoting a list of individual toctrees for that document - and in-turn each part should have a `sections` key - denoting a list child documents/URLs.
+Document pages can then have a `parts` key - denoting a list of individual toctrees for that document - and in-turn each part should have a `sections` key - denoting a list of children links, that are one of: `doc`, `url` or `glob`:
+
+- `doc`: relating to a single document page (as above)
+- `glob`: relating to one or more document pages *via* Unix shell-style wildcards (similar to [`fnmatch`](https://docs.python.org/3/library/fnmatch.html), but single stars don't match slashes.)
+- `url`: relating to an external URL (`http` or `https`)
+
 This can proceed recursively to any depth.
 
 ```yaml
@@ -51,6 +56,7 @@ main:
       - sections:
         - doc: doc2
         - url: https://example.com
+        - glob: other*
 ```
 
 As a shorthand, the `sections` key can be at the same level as the `doc`, which denotes a document with a single `part`.
@@ -64,6 +70,7 @@ main:
     sections:
     - doc: doc2
     - url: https://example.com
+    - glob: other*
 ```
 
 ### Titles and Captions
@@ -166,6 +173,20 @@ To build a template site from only a ToC file:
 $ sphinx-etoc create-site -p path/to/site -e rst path/to/_toc.yml
 ```
 
+Note, when using `glob` you can also add additional files in `meta`/`create_additional`, e.g.
+
+```yaml
+main:
+  doc: intro
+  sections:
+  - glob: doc*
+meta:
+  create_additional:
+  - doc1
+  - doc2
+  - doc3
+```
+
 ## Development Notes
 
 Want to have a built-in CLI including commands:
@@ -187,11 +208,10 @@ Questions / TODOs:
 - What if toctree directive found in file? (raise incompatibility error/warnings)
 - Should `titlesonly` default to `True` (as in jupyter-book)?
 - nested numbered toctree not allowed (logs warning), so should be handled if `numbered: true` is in defaults
-- Handle globbing in sections (separate `glob` key?), also deal with in `create_site_from_toc`
 - Add additional top-level keys, e.g. `appendices` and `bibliography`
 - testing against Windows
 - option to add files not in toc to `ignore_paths` (including glob)
-
+- Add tests for "bad" toc files
 
 [github-ci]: https://github.com/executablebooks/sphinx-external-toc/workflows/continuous-integration/badge.svg?branch=main
 [github-link]: https://github.com/executablebooks/sphinx-external-toc

sphinx_external_toc/api.py

@@ -8,6 +8,14 @@
 import yaml
 
 
+class RefItem(str):
+    """A document name in a toctree list."""
+
+
+class GlobItem(str):
+    """A document glob in a toctree list."""
+
+
 @attr.s(slots=True)
 class UrlItem:
     """A URL in a toctree."""
@@ -21,8 +29,10 @@ class TocItem:
     """An individual toctree within a document."""
 
     # TODO validate uniqueness of docnames (at least one item)
-    sections: List[Union[str, UrlItem]] = attr.ib(
-        validator=deep_iterable(instance_of((str, UrlItem)), instance_of(list))
+    sections: List[Union[GlobItem, RefItem, UrlItem]] = attr.ib(
+        validator=deep_iterable(
+            instance_of((GlobItem, RefItem, UrlItem)), instance_of(list)
+        )
     )
     caption: Optional[str] = attr.ib(None, validator=optional(instance_of(str)))
     numbered: Union[bool, int] = attr.ib(False, validator=instance_of((bool, int)))
@@ -31,7 +41,7 @@ class TocItem:
     reversed: bool = attr.ib(False, validator=instance_of(bool))
 
     def docnames(self) -> List[str]:
-        return [section for section in self.sections if isinstance(section, str)]
+        return [section for section in self.sections if isinstance(section, RefItem)]
 
 
 @attr.s(slots=True)
@@ -53,15 +63,22 @@ def children(self) -> List[str]:
 class SiteMap(MutableMapping):
     """A mapping of documents to their toctrees (or None if terminal)."""
 
-    def __init__(self, root: DocItem) -> None:
+    def __init__(self, root: DocItem, meta: Optional[Dict[str, Any]] = None) -> None:
         self._docs: Dict[str, DocItem] = {}
         self[root.docname] = root
         self._root: DocItem = root
+        self._meta: Dict[str, Any] = meta or {}
 
     @property
     def root(self) -> DocItem:
+        """Return the root document."""
         return self._root
 
+    @property
+    def meta(self) -> Dict[str, Any]:
+        """Return the site-map metadata."""
+        return self._meta
+
     def __getitem__(self, docname: str) -> DocItem:
         return self._docs[docname]
 
@@ -80,10 +97,29 @@ def __iter__(self) -> Iterator[str]:
     def __len__(self) -> int:
         return len(self._docs)
 
-    def as_json(self, root_key: str = "_root") -> Dict[str, Any]:
-        dct = {k: attr.asdict(v) if v else v for k, v in self._docs.items()}
+    @staticmethod
+    def _serializer(inst: Any, field: attr.Attribute, value: Any) -> Any:
+        """Serialize to JSON compatible value.
+
+        (parsed to ``attr.asdict``)
+        """
+        if isinstance(value, (GlobItem, RefItem)):
+            return str(value)
+        return value
+
+    def as_json(
+        self, root_key: str = "_root", meta_key: str = "_meta"
+    ) -> Dict[str, Any]:
+        """Return JSON serialized site-map representation."""
+        dct = {
+            k: attr.asdict(v, value_serializer=self._serializer) if v else v
+            for k, v in self._docs.items()
+        }
         assert root_key not in dct
         dct[root_key] = self.root.docname
+        if self.meta:
+            assert meta_key not in dct
+            dct[meta_key] = self.meta
         return dct
 
 
@@ -107,7 +143,7 @@ def parse_toc_data(data: Dict[str, Any]) -> SiteMap:
 
     doc_item, docs_list = _parse_doc_item(data["main"], defaults, "main/")
 
-    site_map = SiteMap(root=doc_item)
+    site_map = SiteMap(root=doc_item, meta=data.get("meta"))
 
     _parse_docs_list(docs_list, site_map, defaults, "main/")
 
@@ -131,13 +167,13 @@ def _parse_doc_item(
     if not isinstance(parts_data, Sequence):
         raise MalformedError(f"'parts' not a sequence: '{path}'")
 
-    _known_link_keys = {"url", "doc"}
+    _known_link_keys = {"url", "doc", "glob"}
 
     parts = []
     for part_idx, part in enumerate(parts_data):
 
         # generate sections list
-        sections: List[Union[str, UrlItem]] = []
+        sections: List[Union[GlobItem, RefItem, UrlItem]] = []
         for sect_idx, section in enumerate(part["sections"]):
             link_keys = _known_link_keys.intersection(section)
             if not link_keys:
@@ -150,10 +186,12 @@ def _parse_doc_item(
                     "toctree section contains incompatible keys "
                     f"{link_keys!r}: {path}{part_idx}/{sect_idx}"
                 )
-            if link_keys == {"url"}:
+            if link_keys == {"doc"}:
+                sections.append(RefItem(section["doc"]))
+            elif link_keys == {"glob"}:
+                sections.append(GlobItem(section["glob"]))
+            elif link_keys == {"url"}:
                 sections.append(UrlItem(section["url"], section.get("title")))
-            else:
-                sections.append(section["doc"])
 
         # generate toc key-word arguments
         keywords = {}

sphinx_external_toc/events.py

@@ -9,10 +9,10 @@
 from sphinx.environment import BuildEnvironment
 from sphinx.errors import ExtensionError
 from sphinx.transforms import SphinxTransform
-from sphinx.util import logging
-from sphinx.util.matching import Matcher
+from sphinx.util import docname_join, logging
+from sphinx.util.matching import Matcher, patfilter
 
-from .api import DocItem, SiteMap, UrlItem, parse_toc_file
+from .api import DocItem, GlobItem, RefItem, SiteMap, UrlItem, parse_toc_file
 
 logger = logging.getLogger(__name__)
 
@@ -64,7 +64,10 @@ def append_toctrees(app: Sphinx, doctree: document) -> None:
     if doc_item is None or not doc_item.parts:
         return
 
+    # initial variables
     suffixes = app.config.source_suffix
+    all_docnames = app.env.found_docs.copy()
+    all_docnames.remove(app.env.docname)  # remove current document
     excluded = Matcher(app.config.exclude_patterns)
 
     for toctree in doc_item.parts:
@@ -79,7 +82,7 @@ def append_toctrees(app: Sphinx, doctree: document) -> None:
         # TODO this wasn't in the original code,
         # but alabaster theme intermittently raised `KeyError('rawcaption')`
         subnode["rawcaption"] = toctree.caption or ""
-        subnode["glob"] = False
+        subnode["glob"] = any(isinstance(entry, GlobItem) for entry in toctree.sections)
         subnode["hidden"] = True
         subnode["includehidden"] = False
         subnode["numbered"] = toctree.numbered
@@ -92,33 +95,46 @@ def append_toctrees(app: Sphinx, doctree: document) -> None:
         for entry in toctree.sections:
 
             if isinstance(entry, UrlItem):
+
                 subnode["entries"].append((entry.title, entry.url))
-                continue
 
-            child_doc_item = site_map[entry]
+            elif isinstance(entry, RefItem):
 
-            docname = child_doc_item.docname
-            title = child_doc_item.title
+                child_doc_item = site_map[entry]
+                docname = docname_join(app.env.docname, str(entry))
+                title = child_doc_item.title
 
-            # remove any suffixes
-            for suffix in suffixes:
-                if docname.endswith(suffix):
-                    docname = docname[: -len(suffix)]
-                    break
+                # remove any suffixes
+                for suffix in suffixes:
+                    if docname.endswith(suffix):
+                        docname = docname[: -len(suffix)]
+                        break
 
-            if docname not in app.env.found_docs:
-                if excluded(app.env.doc2path(docname, None)):
-                    message = (
-                        f"toctree contains reference to excluded document {docname!r}"
-                    )
-                else:
-                    message = f"toctree contains reference to nonexisting document {docname!r}"
+                if docname not in app.env.found_docs:
+                    if excluded(app.env.doc2path(docname, None)):
+                        message = f"toctree contains reference to excluded document {docname!r}"
+                    else:
+                        message = f"toctree contains reference to nonexisting document {docname!r}"
 
-                node_list.append(doctree.reporter.warning(message))
-                app.env.note_reread()
-            else:
-                subnode["entries"].append((title, docname))
-                subnode["includefiles"].append(docname)
+                    node_list.append(doctree.reporter.warning(message))
+                    app.env.note_reread()
+                else:
+                    subnode["entries"].append((title, docname))
+                    subnode["includefiles"].append(docname)
+
+            elif isinstance(entry, GlobItem):
+                patname = docname_join(app.env.docname, str(entry))
+                docnames = sorted(patfilter(all_docnames, patname))
+                for docname in docnames:
+                    all_docnames.remove(docname)  # don't include it again
+                    subnode["entries"].append((None, docname))
+                    subnode["includefiles"].append(docname)
+                if not docnames:
+                    node_list.append(
+                        doctree.reporter.warning(
+                            f"toctree glob pattern '{entry}' didn't match any documents"
+                        )
+                    )
 
         # reversing entries can be useful when globbing
         if toctree.reversed:

sphinx_external_toc/tools.py

@@ -1,3 +1,4 @@
+from itertools import chain
 from pathlib import Path, PurePosixPath
 from os import linesep
 from typing import Union
@@ -15,6 +16,9 @@ def create_site_from_toc(
 ) -> Path:
     """Create the files defined in the external toc file.
 
+    Additional files can also be created by defining them in
+    `meta`/`create_additional` of the toc
+
     :param toc_path: Path to ToC.
     :param root_path: The root directory , or use ToC file directory.
     :param default_ext: The default file extension to use.
@@ -28,7 +32,7 @@ def create_site_from_toc(
 
     root_path = Path(toc_path).parent if root_path is None else Path(root_path)
 
-    for docname in site_map:
+    for docname in chain(site_map, site_map.meta.get("create_additional", [])):
         if not any(docname.endswith(ext) for ext in {".rst", ".md"}):
             docname += default_ext
         docpath = root_path.joinpath(PurePosixPath(docname))

tests/_toc_files/glob.yml

@@ -0,0 +1,10 @@
+main:
+  doc: intro
+  title: Introduction
+  sections:
+  - glob: doc*
+meta:
+  create_additional:
+  - doc1
+  - doc2
+  - doc3

tests/test_api/test_file_to_sitemap_glob_.yml

@@ -0,0 +1,16 @@
+_meta:
+  create_additional:
+  - doc1
+  - doc2
+  - doc3
+_root: intro
+intro:
+  docname: intro
+  parts:
+  - caption: null
+    numbered: false
+    reversed: false
+    sections:
+    - doc*
+    titlesonly: true
+  title: Introduction

tests/test_tools/test_file_to_sitemap_glob_.yml

@@ -0,0 +1,4 @@
+- doc1.rst
+- doc2.rst
+- doc3.rst
+- intro.rst