Merge pull request #19 from metaobjectsdev/fix/py-codegen-package-init

feat(python-codegen): emit package __init__.py so the out dir is importable

Author: dmealing

docs/features/templates-and-payloads.md

@@ -296,22 +296,28 @@ string output = Renderer.Render(new RenderRequest(
 ### Python
 
 `metaobjects.render` ships the Mustache engine + `Verify`. The Python loader
-recognizes `template.*` + `origin.*`. Payload-VO codegen is not yet emitted —
-consumers pass a `dict` (or any pystache-compatible mapping).
+recognizes `template.*` + `origin.*`. Payload-VO codegen **is** emitted (the
+`payload` generator emits a Pydantic `BaseModel` per template, origin-aware — see
+[Output parsing (FR-006)](#output-parsing-fr-006)), so a consumer can render from
+the generated payload type or from a plain `dict`.
+
+`render` takes a `RenderRequest` (only `payload` + `provider` are required; `ref`
+defaults to `None`, `format` to `"text"`):
 
 ```python
-from metaobjects.render import render, FilesystemProvider
+from metaobjects.render import FilesystemProvider
+from metaobjects.render.renderer import render, RenderRequest
 
-out = render(
-    ref="lobby/welcome",
+out = render(RenderRequest(
     payload={
         "displayName": "Ada",
         "postCount": 12,
         "posts": [{"title": "Hello"}],
     },
     provider=FilesystemProvider("./prompts"),
+    ref="lobby/welcome",
     format="xml",
-)
+))
 ```
 
 ## Output parsing (FR-006)

docs/ports/python.md

@@ -2,8 +2,10 @@
 
 Targets Python 3.11+ on the SQLAlchemy + FastAPI / Pydantic stack. Ships the
 metadata loader (canonical JSON + sigil-free YAML), conformance runner over the
-shared corpora, the FR-004 render engine, and the entity-model codegen. The
-persistence + migration tier is in progress.
+shared corpora, the FR-004 render engine, and the entity / payload / router
+codegen. Schema migrations are owned by the Node `meta` CLI (ADR-0015) — the
+Python port has no `migrate` command by design; it consumes the canonical
+`schema.postgres.sql` artifact verbatim.
 
 ## Install
 
@@ -168,19 +170,23 @@ print(name_field.attr_string_or_none("maxLength"))   # → "200"
 
 ## FR-004 — render
 
+`render` takes a `RenderRequest` (only `payload` + `provider` are required; `ref`
+defaults to `None`, `format` to `"text"`):
+
 ```python
-from metaobjects.render import render, FilesystemProvider
+from metaobjects.render import FilesystemProvider
+from metaobjects.render.renderer import render, RenderRequest
 
-out = render(
-    ref="lobby/welcome",
+out = render(RenderRequest(
     payload={
         "displayName": "Ada",
         "postCount": 12,
         "posts": [{"title": "Hello"}],
     },
     provider=FilesystemProvider("./prompts"),
+    ref="lobby/welcome",
     format="xml",
-)
+))
 ```
 
 `metaobjects.render.verify` drift-checks every `template.*` against its
@@ -284,7 +290,7 @@ import lines are stable.
 | Templates + render (FR-004) | Yes (`metaobjects.render`) |
 | Payload-VO codegen | Yes (`payload_vo_generator` — Pydantic v2 `BaseModel` per template, origin-aware) |
 | Output parser codegen (FR-006) | Yes (`output_parser_generator` — Pydantic throw-only; imports the payload class from the sibling payload module) |
-| Migrations | In progress (`python -m metaobjects.migrate` planned) |
+| Migrations | TS-only by design (ADR-0015) — no Python `migrate` command; consume the canonical `schema.postgres.sql` |
 | Drift verify | Yes — template / payload drift (`metaobjects.render.verify`) |
 | Runtime metadata | Loader API + render engine; SQLAlchemy ObjectManager-equivalent on the roadmap |
 

docs/superpowers/specs/2026-06-13-python-codegen-package-init-emission.md

@@ -0,0 +1,73 @@
+# Python codegen emits a package `__init__.py` so the out dir is importable
+
+_Status: DELIVERED (Python port). Date: 2026-06-13._
+
+## Problem
+
+The Python codegen suite emits modules that use **package-relative imports** — e.g.
+the output-parser imports its payload sibling with `from .relevance_verdict_payload
+import RelevanceVerdictPayload`. A relative import only resolves when the module is
+imported as part of a package, which requires an `__init__.py` in the output
+directory.
+
+The `gen` command does **not** emit that `__init__.py`. So a downstream Python
+consumer that runs `metaobjects gen --out <dir>` and then tries to
+`from <pkg>.<dir>.<name>_output_parser import parse_<name>` hits
+`ImportError: attempted relative import with no known parent package`.
+
+The obvious workaround — the consumer hand-adds an `__init__.py` to the out dir —
+collides with the **codegen-drift gate**: `verify --codegen` regenerates into a temp
+dir and diffs file-for-file, so the hand-added `__init__.py` is reported as
+`extra:   __init__.py` and the gate fails (exit 1). The consumer is stuck between
+"can't import" and "drift gate fails".
+
+This is the same friction whether the generated artifacts are entities, payloads, or
+output parsers — anything that emits a package-relative import.
+
+## Decision
+
+`run_gen` emits an **`@generated` `__init__.py`** in every directory that received a
+generated file (and the intermediate directories up to `out_dir`). Chosen over the
+alternative (teach `verify --codegen` to ignore a consumer-managed `__init__.py`)
+because it keeps the out dir **fully codegen-owned** — the repo's standing principle
+that generated output is disposable and the consumer hand-edits nothing. As a
+generated file, the `__init__.py` is part of the regenerated set, so
+`verify --codegen` passes naturally.
+
+`__init__.py` is Python-specific, so this is a Python-port behavior; no cross-port
+contract change. (Open question for maintainers: whether the sibling ports want an
+analogous "package marker" concept — none is needed for the flat-module ports today.)
+
+## Design
+
+- New `GenConfig.emit_package_init: bool = True` (mirrors the existing
+  `emit_abstract_shapes` toggle). Default-on for the ergonomic path; a consumer that
+  manages its own package layout sets it `False`.
+- In `run_gen`, after the generator loop builds the emitted-file map, compute the set
+  of output-relative directories that received a file plus their ancestors down to
+  `out_dir`, and add an `__init__.py` for each that a generator did not already emit.
+- The marker file carries the standard `@generated by metaobjects` header
+  (`constants.generated_package_init()`), so it flows through the existing
+  `decide_and_write` overwrite policy: regen overwrites it, but a **hand-authored**
+  `__init__.py` (no marker) is **refused**, never clobbered.
+
+## Tests (`tests/codegen/test_runner.py`)
+
+- `test_run_gen_emits_importable_package_init` — out dir gets an `@generated`
+  `__init__.py`.
+- `test_run_gen_package_init_can_be_disabled` — `emit_package_init=False` suppresses it.
+- `test_run_gen_does_not_clobber_handwritten_package_init` — a marker-less hand-authored
+  `__init__.py` is `refused` and left untouched.
+- Existing `test_run_gen_writes_a_model_per_entity` / `_refuses_handwritten_file` updated
+  to assert per-file status (the file map now also contains the package marker).
+
+Full non-integration suite green (the one unrelated failure,
+`ERR_RELATIVE_REF_IN_CANONICAL` missing from the Python `ErrorCode` enum, is FR-026
+Python-port follow-up, independent of this change).
+
+## Follow-ups (not in this change)
+
+- The `__init__.py` is a bare marker. A future enhancement could re-export each
+  module's public symbols for flatter consumer imports (`from <dir> import
+  parse_<name>`), but that needs the generator to know the public surface across
+  files — deferred.

server/python/src/metaobjects/codegen/config.py

@@ -9,6 +9,7 @@ class GenConfig:
     out_dir: str
     output_layout: str = "flat"  # "flat" only in sub-project A
     emit_abstract_shapes: bool = True  # Python concretes subclass the abstract base model
+    emit_package_init: bool = True  # emit an @generated __init__.py so the out dir imports as a package
     # FR-019 (ADR-0026): per-port resolution of a @provided enum's import module. The module
     # never lives in metadata (ADR-0001) — it is codegen config. ``provided_enum_packages``
     # maps a declaring metadata package ("acme::shop") to the Python import module; with a

server/python/src/metaobjects/codegen/constants.py

@@ -11,3 +11,18 @@ def generated_header(entity_name: str, fqn: str) -> str:
         f"# Source metadata: {entity_name} ({fqn})\n"
         f"# Customize via {entity_name}_extra.py in this directory.\n"
     )
+
+
+def generated_package_init() -> str:
+    """Content for a generated package ``__init__.py``.
+
+    A bare marker file that makes the codegen out dir an importable Python package —
+    the emitted modules use package-relative imports (``from .x import ...``), so the
+    consumer needs the package marker to import them. Carries the @generated marker so
+    the overwrite policy re-writes it on regen and refuses to clobber a hand-authored
+    ``__init__.py`` (which lacks the marker).
+    """
+    return (
+        f"# {GENERATED_MARKER} — DO NOT EDIT.\n"
+        f"# Package marker so the generated output directory is importable.\n"
+    )

server/python/src/metaobjects/codegen/runner.py

@@ -11,6 +11,7 @@
 from metaobjects.meta.core.object.meta_object import MetaObject
 from metaobjects.shared.base_types import TYPE_OBJECT
 from .config import GenConfig
+from .constants import generated_package_init
 from .generator import GenContext, Generator
 from .overwrite_policy import decide_and_write
 
@@ -100,6 +101,25 @@ def run_gen(
                 )
             emitted[full] = (f.content, gen.name)
 
+    # Make the out dir an importable package: emit an @generated __init__.py in
+    # every directory that received a generated file (and the intermediate dirs up
+    # to out_dir). The generated modules use package-relative imports, so without
+    # this a consumer can't import them. A generator-emitted __init__.py wins; a
+    # hand-authored one is left untouched by the overwrite policy below.
+    if config.emit_package_init:
+        pkg_rel_dirs: set[str] = set()
+        for full in list(emitted):
+            d = os.path.dirname(os.path.relpath(full, config.out_dir))
+            while True:
+                pkg_rel_dirs.add(d)
+                if d == "":
+                    break
+                d = os.path.dirname(d)
+        for d in pkg_rel_dirs:
+            init_full = os.path.join(config.out_dir, d, "__init__.py")
+            if init_full not in emitted:
+                emitted[init_full] = (generated_package_init(), "package-init")
+
     for full, (content, _by) in emitted.items():
         status = decide_and_write(full, content, merge_strategy)
         result.files.append((full, status))

server/python/tests/codegen/test_runner.py

@@ -6,6 +6,7 @@
 from metaobjects import MetaDataLoader
 from metaobjects.meta.meta_data import MetaData
 from metaobjects.codegen.config import GenConfig
+from metaobjects.codegen.constants import GENERATED_MARKER
 from metaobjects.codegen.runner import run_gen
 from metaobjects.codegen.generators.entity_model import entity_model
 
@@ -26,7 +27,51 @@ def test_run_gen_writes_a_model_per_entity(tmp_path: Path) -> None:
     result = run_gen(GenConfig(out_dir=str(out)), root, generators=[entity_model()])
     assert (out / "Subscriber.py").exists()
     assert "class Subscriber(BaseModel):" in (out / "Subscriber.py").read_text()
-    assert [w[1] for w in result.files] == ["new"]
+    statuses = {Path(p).name: s for p, s in result.files}
+    assert statuses["Subscriber.py"] == "new"
+    assert statuses["__init__.py"] == "new"  # package marker emitted alongside
+
+
+def test_run_gen_emits_importable_package_init(tmp_path: Path) -> None:
+    """The codegen out dir is a self-contained package: run_gen emits an
+    ``@generated`` ``__init__.py`` so a consumer can import the generated modules
+    (which use package-relative imports) without hand-adding a marker file that
+    ``verify --codegen`` would then flag as ``extra``."""
+    root = _load(tmp_path / "meta", {"metadata.root": {"package": "acme", "children": [
+        {"object.entity": {"name": "Subscriber", "children": [
+            {"field.string": {"name": "email", "@required": True}},
+        ]}},
+    ]}})
+    out = tmp_path / "out"
+    run_gen(GenConfig(out_dir=str(out)), root, generators=[entity_model()])
+    init = out / "__init__.py"
+    assert init.exists(), "expected a generated __init__.py in the out dir"
+    assert GENERATED_MARKER in init.read_text(), "__init__.py must carry the @generated marker"
+
+
+def test_run_gen_package_init_can_be_disabled(tmp_path: Path) -> None:
+    """``emit_package_init=False`` suppresses the package marker (consumer owns it)."""
+    root = _load(tmp_path / "meta", {"metadata.root": {"package": "acme", "children": [
+        {"object.entity": {"name": "Subscriber", "children": [{"field.string": {"name": "x"}}]}},
+    ]}})
+    out = tmp_path / "out"
+    run_gen(GenConfig(out_dir=str(out), emit_package_init=False), root,
+            generators=[entity_model()])
+    assert not (out / "__init__.py").exists()
+
+
+def test_run_gen_does_not_clobber_handwritten_package_init(tmp_path: Path) -> None:
+    """A hand-authored ``__init__.py`` (no @generated marker) is left untouched."""
+    root = _load(tmp_path / "meta", {"metadata.root": {"package": "acme", "children": [
+        {"object.entity": {"name": "Subscriber", "children": [{"field.string": {"name": "x"}}]}},
+    ]}})
+    out = tmp_path / "out"
+    out.mkdir()
+    (out / "__init__.py").write_text("# hand written package init\nVERSION = '1'\n")
+    result = run_gen(GenConfig(out_dir=str(out)), root, generators=[entity_model()])
+    statuses = {Path(p).name: s for p, s in result.files}
+    assert statuses["__init__.py"] == "refused"
+    assert "hand written package init" in (out / "__init__.py").read_text()  # untouched
 
 
 def test_run_gen_skips_unsafe_names_with_warning(tmp_path: Path) -> None:
@@ -49,7 +94,8 @@ def test_run_gen_refuses_handwritten_file(tmp_path: Path) -> None:
     out.mkdir()
     (out / "Subscriber.py").write_text("# hand written, no marker\nx = 1\n")
     result = run_gen(GenConfig(out_dir=str(out)), root, generators=[entity_model()])
-    assert [w[1] for w in result.files] == ["refused"]
+    statuses = {Path(p).name: s for p, s in result.files}
+    assert statuses["Subscriber.py"] == "refused"
     assert any("Refused to overwrite" in w for w in result.warnings)
     assert "hand written" in (out / "Subscriber.py").read_text()  # untouched