Swift: some restructuring of codegen

Loading of the schema and dbscheme has been moved to a separate
`loaders` package for better separation of concerns.

Author: redsun82

swift/codegen/generators/BUILD.bazel

@@ -6,6 +6,6 @@ py_library(
     visibility = ["//swift/codegen:__subpackages__"],
     deps = [
         "//swift/codegen/lib",
-        requirement("toposort"),
+        "//swift/codegen/loaders",
     ],
 )

swift/codegen/generators/cppgen.py

@@ -17,6 +17,7 @@
 import inflection
 
 from swift.codegen.lib import cpp, schema
+from swift.codegen.loaders import schemaloader
 
 
 def _get_type(t: str, add_or_none_except: typing.Optional[str] = None) -> str:
@@ -90,7 +91,7 @@ def get_classes(self):
 
 def generate(opts, renderer):
     assert opts.cpp_output
-    processor = Processor(schema.load_file(opts.schema))
+    processor = Processor(schemaloader.load_file(opts.schema))
     out = opts.cpp_output
     for dir, classes in processor.get_classes().items():
         renderer.render(cpp.ClassList(classes, opts.schema,

swift/codegen/generators/dbschemegen.py

@@ -18,8 +18,8 @@
 import inflection
 
 from swift.codegen.lib import schema
+from swift.codegen.loaders import schemaloader
 from swift.codegen.lib.dbscheme import *
-from typing import Set, List
 
 log = logging.getLogger(__name__)
 
@@ -123,7 +123,7 @@ def generate(opts, renderer):
     input = opts.schema
     out = opts.dbscheme
 
-    data = schema.load_file(input)
+    data = schemaloader.load_file(input)
 
     dbscheme = Scheme(src=input.relative_to(opts.swift_dir),
                       includes=get_includes(data, include_dir=input.parent, swift_dir=opts.swift_dir),

swift/codegen/generators/qlgen.py

@@ -30,6 +30,7 @@
 import inflection
 
 from swift.codegen.lib import schema, ql
+from swift.codegen.loaders import schemaloader
 
 log = logging.getLogger(__name__)
 
@@ -297,7 +298,7 @@ def generate(opts, renderer):
 
     stubs = {q for q in stub_out.rglob("*.qll")}
 
-    data = schema.load_file(input)
+    data = schemaloader.load_file(input)
 
     classes = {name: get_ql_class(cls) for name, cls in data.classes.items()}
     if not classes:

swift/codegen/generators/trapgen.py

@@ -18,6 +18,7 @@
 from toposort import toposort_flatten
 
 from swift.codegen.lib import dbscheme, cpp
+from swift.codegen.loaders import dbschemeloader
 
 log = logging.getLogger(__name__)
 
@@ -73,7 +74,7 @@ def generate(opts, renderer):
     out = opts.cpp_output
 
     traps = {pathlib.Path(): []}
-    for e in dbscheme.iterload(opts.dbscheme):
+    for e in dbschemeloader.iterload(opts.dbscheme):
         if e.is_table:
             traps.setdefault(e.dir, []).append(get_trap(e))
         elif e.is_union:

swift/codegen/lib/BUILD.bazel

@@ -2,11 +2,10 @@ load("@swift_codegen_deps//:requirements.bzl", "requirement")
 
 py_library(
     name = "lib",
-    srcs = glob(["**/*.py"]),
+    srcs = glob(["*.py"]),
     visibility = ["//swift/codegen:__subpackages__"],
     deps = [
         requirement("pystache"),
-        requirement("pyyaml"),
         requirement("inflection"),
     ],
 )

swift/codegen/lib/dbscheme.py

@@ -105,54 +105,3 @@ class Scheme:
     src: str
     includes: List[SchemeInclude]
     declarations: List[Decl]
-
-
-class Re:
-    entity = re.compile(
-        "(?m)"
-        r"(?:^#keyset\[(?P<tablekeys>[\w\s,]+)\][\s\n]*)?^(?P<table>\w+)\("
-        r"(?:\s*//dir=(?P<tabledir>\S*))?(?P<tablebody>[^\)]*)"
-        r"\);?"
-        "|"
-        r"^(?P<union>@\w+)\s*=\s*(?P<unionbody>@\w+(?:\s*\|\s*@\w+)*)\s*;?"
-    )
-    field = re.compile(r"(?m)[\w\s]*\s(?P<field>\w+)\s*:\s*(?P<type>@?\w+)(?P<ref>\s+ref)?")
-    key = re.compile(r"@\w+")
-    comment = re.compile(r"(?m)(?s)/\*.*?\*/|//(?!dir=)[^\n]*$")  # lookahead avoid ignoring metadata like //dir=foo
-
-
-def get_column(match):
-    return Column(
-        schema_name=match["field"].rstrip("_"),
-        type=match["type"],
-        binding=not match["ref"],
-    )
-
-
-def get_table(match):
-    keyset = None
-    if match["tablekeys"]:
-        keyset = KeySet(k.strip() for k in match["tablekeys"].split(","))
-    return Table(
-        name=match["table"],
-        columns=[get_column(f) for f in Re.field.finditer(match["tablebody"])],
-        keyset=keyset,
-        dir=pathlib.PosixPath(match["tabledir"]) if match["tabledir"] else None,
-    )
-
-
-def get_union(match):
-    return Union(
-        lhs=match["union"],
-        rhs=(d[0] for d in Re.key.finditer(match["unionbody"])),
-    )
-
-
-def iterload(file):
-    with open(file) as file:
-        data = Re.comment.sub("", file.read())
-    for e in Re.entity.finditer(data):
-        if e["table"]:
-            yield get_table(e)
-        elif e["union"]:
-            yield get_union(e)

swift/codegen/lib/schema/schema.py swift/codegen/lib/schema.pyswift/codegen/lib/schema/schema.py renamed to swift/codegen/lib/schema.py

@@ -1,15 +1,9 @@
-""" schema.yml format representation """
-import pathlib
-import re
-import types
+""" schema format representation """
 import typing
 from dataclasses import dataclass, field
 from typing import List, Set, Union, Dict, Optional
 from enum import Enum, auto
 import functools
-import importlib.util
-from toposort import toposort_flatten
-import inflection
 
 
 class Error(Exception):
@@ -198,125 +192,3 @@ def split_doc(doc):
     while trimmed and not trimmed[0]:
         trimmed.pop(0)
     return trimmed
-
-
-@dataclass
-class _PropertyNamer(PropertyModifier):
-    name: str
-
-    def modify(self, prop: Property):
-        prop.name = self.name.rstrip("_")
-
-
-def _get_class(cls: type) -> Class:
-    if not isinstance(cls, type):
-        raise Error(f"Only class definitions allowed in schema, found {cls}")
-    # we must check that going to dbscheme names and back is preserved
-    # In particular this will not happen if uppercase acronyms are included in the name
-    to_underscore_and_back = inflection.camelize(inflection.underscore(cls.__name__), uppercase_first_letter=True)
-    if cls.__name__ != to_underscore_and_back:
-        raise Error(f"Class name must be upper camel-case, without capitalized acronyms, found {cls.__name__} "
-                    f"instead of {to_underscore_and_back}")
-    if len({b._group for b in cls.__bases__ if hasattr(b, "_group")}) > 1:
-        raise Error(f"Bases with mixed groups for {cls.__name__}")
-    if any(getattr(b, "_null", False) for b in cls.__bases__):
-        raise Error(f"Null class cannot be derived")
-    return Class(name=cls.__name__,
-                 bases=[b.__name__ for b in cls.__bases__ if b is not object],
-                 derived={d.__name__ for d in cls.__subclasses__()},
-                 # getattr to inherit from bases
-                 group=getattr(cls, "_group", ""),
-                 # in the following we don't use `getattr` to avoid inheriting
-                 pragmas=cls.__dict__.get("_pragmas", []),
-                 ipa=cls.__dict__.get("_ipa", None),
-                 properties=[
-                     a | _PropertyNamer(n)
-                     for n, a in cls.__dict__.get("__annotations__", {}).items()
-                 ],
-                 doc=split_doc(cls.__doc__),
-                 default_doc_name=cls.__dict__.get("_doc_name"),
-                 )
-
-
-def _toposort_classes_by_group(classes: typing.Dict[str, Class]) -> typing.Dict[str, Class]:
-    groups = {}
-    ret = {}
-
-    for name, cls in classes.items():
-        groups.setdefault(cls.group, []).append(name)
-
-    for group, grouped in sorted(groups.items()):
-        inheritance = {name: classes[name].bases for name in grouped}
-        for name in toposort_flatten(inheritance):
-            ret[name] = classes[name]
-
-    return ret
-
-
-def _fill_ipa_information(classes: typing.Dict[str, Class]):
-    """ Take a dictionary where the `ipa` field is filled for all explicitly synthesized classes
-    and update it so that all non-final classes that have only synthesized final descendants
-    get `True` as` value for the `ipa` field
-    """
-    if not classes:
-        return
-
-    is_ipa: typing.Dict[str, bool] = {}
-
-    def fill_is_ipa(name: str):
-        if name not in is_ipa:
-            cls = classes[name]
-            for d in cls.derived:
-                fill_is_ipa(d)
-            if cls.ipa is not None:
-                is_ipa[name] = True
-            elif not cls.derived:
-                is_ipa[name] = False
-            else:
-                is_ipa[name] = all(is_ipa[d] for d in cls.derived)
-
-    root = next(iter(classes))
-    fill_is_ipa(root)
-
-    for name, cls in classes.items():
-        if cls.ipa is None and is_ipa[name]:
-            cls.ipa = True
-
-
-def load(m: types.ModuleType) -> Schema:
-    includes = set()
-    classes = {}
-    known = {"int", "string", "boolean"}
-    known.update(n for n in m.__dict__ if not n.startswith("__"))
-    import swift.codegen.lib.schema.defs as defs
-    null = None
-    for name, data in m.__dict__.items():
-        if hasattr(defs, name):
-            continue
-        if name == "__includes":
-            includes = set(data)
-            continue
-        if name.startswith("__"):
-            continue
-        cls = _get_class(data)
-        if classes and not cls.bases:
-            raise Error(
-                f"Only one root class allowed, found second root {name}")
-        cls.check_types(known)
-        classes[name] = cls
-        if getattr(data, "_null", False):
-            if null is not None:
-                raise Error(f"Null class {null} already defined, second null class {name} not allowed")
-            null = name
-            cls.is_null_class = True
-
-    _fill_ipa_information(classes)
-
-    return Schema(includes=includes, classes=_toposort_classes_by_group(classes), null=null)
-
-
-def load_file(path: pathlib.Path) -> Schema:
-    spec = importlib.util.spec_from_file_location("schema", path)
-    module = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(module)
-    return load(module)