support modules and apps subfolders; pyscript.config is yaml config

Author: craigbarratt

custom_components/pyscript/__init__.py

@@ -7,14 +7,16 @@
 
 import voluptuous as vol
 
+from homeassistant.config import async_hass_config_yaml, async_process_component_config
 from homeassistant.const import (
     EVENT_HOMEASSISTANT_STARTED,
     EVENT_HOMEASSISTANT_STOP,
     EVENT_STATE_CHANGED,
     SERVICE_RELOAD,
 )
+from homeassistant.exceptions import HomeAssistantError
 import homeassistant.helpers.config_validation as cv
-from homeassistant.loader import bind_hass
+from homeassistant.loader import async_get_integration, bind_hass
 
 from .const import DOMAIN, FOLDER, LOGGER_PATH, SERVICE_JUPYTER_KERNEL_START
 from .eval import AstEval
@@ -30,7 +32,11 @@
 CONF_ALLOW_ALL_IMPORTS = "allow_all_imports"
 
 CONFIG_SCHEMA = vol.Schema(
-    {DOMAIN: vol.Schema({vol.Optional(CONF_ALLOW_ALL_IMPORTS, default=False): cv.boolean})},
+    {
+        DOMAIN: vol.Schema(
+            {vol.Optional(CONF_ALLOW_ALL_IMPORTS, default=False): cv.boolean}, extra=vol.ALLOW_EXTRA,
+        )
+    },
     extra=vol.ALLOW_EXTRA,
 )
 
@@ -44,37 +50,51 @@ async def async_setup(hass, config):
     State.register_functions()
     GlobalContextMgr.init()
 
-    path = hass.config.path(FOLDER)
+    pyscript_folder = hass.config.path(FOLDER)
 
     def check_isdir(path):
         return os.path.isdir(path)
 
-    if not await hass.async_add_executor_job(check_isdir, path):
+    if not await hass.async_add_executor_job(check_isdir, pyscript_folder):
         _LOGGER.error("Folder %s not found in configuration folder", FOLDER)
         return False
 
     hass.data.setdefault(DOMAIN, {})
     hass.data[DOMAIN]["allow_all_imports"] = config[DOMAIN].get(CONF_ALLOW_ALL_IMPORTS)
 
-    await compile_scripts(hass)
+    State.set_pyscript_config(config.get(DOMAIN, {}))
 
-    _LOGGER.debug("adding reload handler")
+    await load_scripts(hass)
 
     async def reload_scripts_handler(call):
         """Handle reload service calls."""
-        _LOGGER.debug("stopping triggers and services, reloading scripts, and restarting")
+        _LOGGER.debug("reload: yaml, reloading scripts, and restarting")
+
+        try:
+            conf = await async_hass_config_yaml(hass)
+        except HomeAssistantError as err:
+            _LOGGER.error(err)
+            return
+
+        integration = await async_get_integration(hass, DOMAIN)
+
+        config = await async_process_component_config(hass, conf, integration)
+
+        # GlobalContext.global_sym_table_add("pyscript.config", config.get(DOMAIN, {}))
+        State.set_pyscript_config(config.get(DOMAIN, {}))
 
         ctx_delete = {}
         for global_ctx_name, global_ctx in GlobalContextMgr.items():
-            if not global_ctx_name.startswith("file."):
+            idx = global_ctx_name.find(".")
+            if idx < 0 or global_ctx_name[0:idx] not in {"file", "apps", "modules"}:
                 continue
             global_ctx.stop()
             global_ctx.set_auto_start(False)
             ctx_delete[global_ctx_name] = global_ctx
         for global_ctx_name, global_ctx in ctx_delete.items():
             await GlobalContextMgr.delete(global_ctx_name)
 
-        await compile_scripts(hass)
+        await load_scripts(hass)
 
         for global_ctx_name, global_ctx in GlobalContextMgr.items():
             if not global_ctx_name.startswith("file."):
@@ -88,7 +108,7 @@ async def jupyter_kernel_start(call):
         _LOGGER.debug("service call to jupyter_kernel_start: %s", call.data)
 
         global_ctx_name = GlobalContextMgr.new_name("jupyter_")
-        global_ctx = GlobalContext(global_ctx_name, hass, global_sym_table={})
+        global_ctx = GlobalContext(global_ctx_name, global_sym_table={}, manager=GlobalContextMgr)
         global_ctx.set_auto_start(True)
 
         GlobalContextMgr.set(global_ctx_name, global_ctx)
@@ -147,44 +167,25 @@ async def stop_triggers(event):
 
 
 @bind_hass
-async def compile_scripts(hass):
-    """Compile all python scripts in FOLDER."""
-
-    path = hass.config.path(FOLDER)
+async def load_scripts(hass):
+    """Load all python scripts in FOLDER."""
 
-    _LOGGER.debug("compile_scripts: path = %s", path)
+    load_paths = [hass.config.path(FOLDER) + "/apps", hass.config.path(FOLDER)]
 
-    def glob_files(path, match):
-        return glob.iglob(os.path.join(path, match))
+    _LOGGER.debug("load_scripts: load_paths = %s", load_paths)
 
-    def read_file(path):
-        with open(path) as file_desc:
-            source = file_desc.read()
-        return source
+    def glob_files(load_paths, match):
+        source_files = []
+        for path in load_paths:
+            source_files += sorted(glob.glob(os.path.join(path, match)))
+        return source_files
 
-    source_files = await hass.async_add_executor_job(glob_files, path, "*.py")
+    source_files = await hass.async_add_executor_job(glob_files, load_paths, "*.py")
 
-    for file in sorted(source_files):
-        _LOGGER.debug("reading and parsing %s", file)
-        name = os.path.splitext(os.path.basename(file))[0]
-        source = await hass.async_add_executor_job(read_file, file)
+    for source_file in sorted(source_files):
+        name = os.path.splitext(os.path.basename(source_file))[0]
 
-        global_ctx_name = f"file.{name}"
-        global_ctx = GlobalContext(global_ctx_name, hass, global_sym_table={})
+        global_ctx = GlobalContext(f"file.{name}", global_sym_table={}, manager=GlobalContextMgr)
         global_ctx.set_auto_start(False)
 
-        ast_ctx = AstEval(global_ctx_name, global_ctx)
-        Function.install_ast_funcs(ast_ctx)
-
-        if not ast_ctx.parse(source, filename=file):
-            exc = ast_ctx.get_exception_long()
-            ast_ctx.get_logger().error(exc)
-            global_ctx.stop()
-            continue
-        await ast_ctx.eval()
-        exc = ast_ctx.get_exception_long()
-        if exc is not None:
-            ast_ctx.get_logger().error(exc)
-            global_ctx.stop()
-            continue
-        GlobalContextMgr.set(global_ctx_name, global_ctx)
+        await GlobalContextMgr.load_file(source_file, global_ctx)

custom_components/pyscript/eval.py

@@ -691,9 +691,7 @@ def __init__(self, name, global_ctx, logger_name=None):
         self.logger_handlers = set()
         self.logger = None
         self.set_logger_name(logger_name if logger_name is not None else self.name)
-        self.allow_all_imports = (
-            Function.hass.data[DOMAIN]["allow_all_imports"] if global_ctx.hass is not None else False
-        )
+        self.allow_all_imports = Function.hass.data.get(DOMAIN, {}).get("allow_all_imports", False)
 
     async def ast_not_implemented(self, arg, *args):
         """Raise NotImplementedError exception for unimplemented AST types."""
@@ -734,22 +732,26 @@ async def ast_module(self, arg):
     async def ast_import(self, arg):
         """Execute import."""
         for imp in arg.names:
-            if not self.allow_all_imports and imp.name not in ALLOWED_IMPORTS:
-                raise ModuleNotFoundError(f"import of {imp.name} not allowed")
-            if imp.name not in sys.modules:
-                mod = await Function.hass.async_add_executor_job(importlib.import_module, imp.name)
-            else:
-                mod = sys.modules[imp.name]
+            mod = await self.global_ctx.module_import(imp.name)
+            if not mod:
+                if not self.allow_all_imports and imp.name not in ALLOWED_IMPORTS:
+                    raise ModuleNotFoundError(f"import of {imp.name} not allowed")
+                if imp.name not in sys.modules:
+                    mod = await Function.hass.async_add_executor_job(importlib.import_module, imp.name)
+                else:
+                    mod = sys.modules[imp.name]
             self.sym_table[imp.name if imp.asname is None else imp.asname] = mod
 
     async def ast_importfrom(self, arg):
         """Execute from X import Y."""
-        if not self.allow_all_imports and arg.module not in ALLOWED_IMPORTS:
-            raise ModuleNotFoundError(f"import from {arg.module} not allowed")
-        if arg.module not in sys.modules:
-            mod = await Function.hass.async_add_executor_job(importlib.import_module, arg.module)
-        else:
-            mod = sys.modules[arg.module]
+        mod = await self.global_ctx.module_import(arg.module)
+        if not mod:
+            if not self.allow_all_imports and arg.module not in ALLOWED_IMPORTS:
+                raise ModuleNotFoundError(f"import from {arg.module} not allowed")
+            if arg.module not in sys.modules:
+                mod = await Function.hass.async_add_executor_job(importlib.import_module, arg.module)
+            else:
+                mod = sys.modules[arg.module]
         for imp in arg.names:
             if imp.name == "*":
                 for name, value in mod.__dict__.items():
@@ -1405,12 +1407,12 @@ async def loopvar_scope_save(self, generators):
         # looping variables are in their own implicit nested scope, so save/restore
         # variables in the current scope with the same names
         #
-        vars = set()
+        lvars = set()
         for gen in generators:
             await self.get_names(
-                ast.Assign(targets=[gen.target], value=ast.Constant(value=None)), local_names=vars
+                ast.Assign(targets=[gen.target], value=ast.Constant(value=None)), local_names=lvars
             )
-        return vars, {var: self.sym_table[var] for var in vars if var in self.sym_table}
+        return lvars, {var: self.sym_table[var] for var in lvars if var in self.sym_table}
 
     async def loopvar_scope_restore(self, var_names, save_vars):
         """Restore current scope variables that match looping target vars."""

custom_components/pyscript/global_ctx.py

@@ -1,8 +1,11 @@
 """Global context handling."""
 
 import logging
+import os
+from types import ModuleType
 
-from .const import LOGGER_PATH
+from .const import FOLDER, LOGGER_PATH
+from .eval import AstEval
 from .function import Function
 from .trigger import TrigInfo
 
@@ -12,15 +15,16 @@
 class GlobalContext:
     """Define class for global variables and trigger context."""
 
-    def __init__(self, name, hass, global_sym_table=None):
+    def __init__(self, name, global_sym_table=None, manager=None):
         """Initialize GlobalContext."""
         self.name = name
-        self.hass = hass
         self.global_sym_table = global_sym_table if global_sym_table else {}
         self.triggers = set()
         self.triggers_delay_start = set()
         self.logger = logging.getLogger(LOGGER_PATH + "." + name)
+        self.manager = manager
         self.auto_start = False
+        self.module = None
 
     def trigger_register(self, func):
         """Register a trigger function; return True if start now."""
@@ -64,6 +68,28 @@ def get_trig_info(self, name, trig_args):
         """Return a new trigger info instance with the given args."""
         return TrigInfo(name, trig_args, self)
 
+    async def module_import(self, module_name):
+        """Import a module from the pyscript/modules folder."""
+        mod_ctx_name = f"module.{module_name}"
+        mod_ctx = self.manager.get(mod_ctx_name)
+        if not mod_ctx:
+
+            def check_isfile(filepath):
+                return os.path.isfile(filepath)
+
+            filepath = Function.hass.config.path(FOLDER) + f"/modules/{module_name}.py"
+            if await Function.hass.async_add_executor_job(check_isfile, filepath):
+                mod = ModuleType(module_name)
+                global_ctx = GlobalContext(mod_ctx_name, global_sym_table=mod.__dict__, manager=self.manager)
+                global_ctx.set_auto_start(True)
+                await self.manager.load_file(filepath, global_ctx)
+                global_ctx.module = mod
+                mod_ctx = self.manager.get(mod_ctx_name)
+
+        if mod_ctx and mod_ctx.module:
+            return mod_ctx.module
+        return None
+
 
 class GlobalContextMgr:
     """Define class for all global contexts."""
@@ -156,3 +182,39 @@ def new_name(cls, root):
             cls.name_seq += 1
             if name not in cls.contexts:
                 return name
+
+    @classmethod
+    async def load_file(cls, filepath, global_ctx):
+        """Load, parse and run the given script file."""
+
+        def read_file(path):
+            try:
+                with open(path) as file_desc:
+                    source = file_desc.read()
+                return source
+            except Exception as exc:
+                _LOGGER.error("%s", exc)
+                return None
+
+        _LOGGER.debug("reading and parsing %s", filepath)
+
+        source = await Function.hass.async_add_executor_job(read_file, filepath)
+
+        if source is None:
+            return
+
+        ast_ctx = AstEval(global_ctx.get_name(), global_ctx)
+        Function.install_ast_funcs(ast_ctx)
+
+        if not ast_ctx.parse(source, filename=filepath):
+            exc = ast_ctx.get_exception_long()
+            ast_ctx.get_logger().error(exc)
+            global_ctx.stop()
+            return
+        await ast_ctx.eval()
+        exc = ast_ctx.get_exception_long()
+        if exc is not None:
+            ast_ctx.get_logger().error(exc)
+            global_ctx.stop()
+            return
+        cls.set(global_ctx.get_name(), global_ctx)

custom_components/pyscript/state.py

@@ -29,6 +29,11 @@ class State:
     #
     notify_var_last = {}
 
+    #
+    # pyscript yaml configuration
+    #
+    pyscript_config = {}
+
     def __init__(self):
         """Warn on State instantiation."""
         _LOGGER.error("State class is not meant to be instantiated")
@@ -170,11 +175,22 @@ async def names(cls, domain=None):
 
     @classmethod
     def register_functions(cls):
-        """Register state functions."""
+        """Register state functions and config variable."""
         functions = {
             "state.get": cls.get,
             "state.set": cls.set,
             "state.names": cls.names,
             "state.get_attr": cls.get_attr,
+            "pyscript.config": cls.pyscript_config,
         }
         Function.register(functions)
+
+    @classmethod
+    def set_pyscript_config(cls, config):
+        """Set pyscript yaml config."""
+        #
+        # have to update inplace, since dist is already used as value
+        #
+        cls.pyscript_config.clear()
+        for name, value in config.items():
+            cls.pyscript_config[name] = value

tests/test_decorator_errors.py

@@ -28,7 +28,7 @@ async def setup_script(hass, notify_q, now, source):
     with patch("homeassistant.loader.async_get_integration", return_value=integration), patch(
         "custom_components.pyscript.os.path.isdir", return_value=True
     ), patch("custom_components.pyscript.glob.iglob", return_value=scripts), patch(
-        "custom_components.pyscript.open", mock_open(read_data=source), create=True,
+        "custom_components.pyscript.global_ctx.open", mock_open(read_data=source), create=True,
     ), patch(
         "custom_components.pyscript.trigger.dt_now", return_value=now
     ):

tests/test_function.py

@@ -114,7 +114,7 @@ async def setup_script(hass, notify_q, now, source):
     with patch("homeassistant.loader.async_get_integration", return_value=integration), patch(
         "custom_components.pyscript.os.path.isdir", return_value=True
     ), patch("custom_components.pyscript.glob.iglob", return_value=scripts), patch(
-        "custom_components.pyscript.open", mock_open(read_data=source), create=True,
+        "custom_components.pyscript.global_ctx.open", mock_open(read_data=source), create=True,
     ), patch(
         "custom_components.pyscript.trigger.dt_now", return_value=now
     ):