ENH(sideffects): DIFFER from regular DATA...

to allow the same name to be used as regular and sideffect data node.

+ ENH: sideffect-strclass now contains its class-name.
+ DOC: a lot, comply with docstrings 1-liners;
  changes also on optionals.
+ ENH(TCs): +x2 elaborate TCs, check SEs differ from data.

Author: ankostis

graphkit/modifiers.py

@@ -11,12 +11,14 @@
 
 class optional(str):
     """
-    Input values in ``needs`` may be designated as optional using this modifier.
-    If this modifier is applied to an input value, that value will be input to
-    the ``operation`` if it is available.  The function underlying the
-    ``operation`` should have a parameter with the same name as the input value
-    in ``needs``, and the input value will be passed as a keyword argument if
-    it is available.
+    An optional need signifies that the function's argument may not receive a value.
+
+    Only input values in ``needs`` may be designated as optional using this modifier.
+    An ``operation`` will receive a value for an optional need only if if it is available
+    in the graph at the time of its invocation.
+    The ``operation``'s function should have a defaulted parameter with the same name
+    as the opetional, and the input value will be passed as a keyword argument,
+    if it is available.
 
     Here is an example of an operation that uses an optional argument::
 
@@ -34,7 +36,7 @@ class optional(str):
         NetworkOperation(name='mygraph',
                          needs=[optional('a'), optional('b'), optional('c')],
                          provides=['sum'])
-        
+
         >>> # The graph works with and without 'c' provided as input.
         >>> graph({'a': 5, 'b': 2, 'c': 4})['sum']
         11
@@ -51,22 +53,25 @@ def __repr__(self):
 
 class sideffect(str):
     """
-    Inputs & outputs in ``needs`` & ``provides`` may be designated as *sideffects*
-    using this modifier.  *Tokens* work as usual while solving the DAG but
-    they are never assigned any values to/from the ``operation`` functions.
-    Specifically:
+    A sideffect data-dependency participates in the graph but never given/asked in functions.
+
+    Both inputs & outputs in ``needs`` & ``provides`` may be designated as *sideffects*
+    using this modifier.  *Sideffects* work as usual while solving the graph but
+    they do not interact with the ``operation``'s function;  specifically:
 
     - input sideffects are NOT fed into the function;
     - output sideffects are NOT expected from the function.
 
-    Their purpose is to describe functions that have modify internal state
-    their arguments ("side-effects").
-    Note that an ``operation`` with just a single *sideffect* output return
-    no value at all, but it would still be called for its side-effects  only.
+    .. info:
+        an ``operation`` with just a single *sideffect* output return no value at all,
+        but it would still be called for its side-effect only.
 
+    Their purpose is to describe operations that modify the internal state of
+    some of their arguments ("side-effects").
     A typical use case is to signify columns required to produce new ones in
     pandas dataframes::
 
+
         >>> from graphkit import operation, compose, sideffect
 
         >>> # Function appending a new dataframe column from two pre-existing ones.
@@ -81,16 +86,42 @@ class sideffect(str):
         ...         provides=[sideffect('sum')])(addcolumns)
         ... )
         >>> graph
-        NetworkOperation(name='mygraph', needs=[optional('df'), optional('a'), optional('b')], provides=[sideffect('sum')])
+        NetworkOperation(name='mygraph', needs=[optional('df'), optional('sideffect(a)'), optional('sideffect(b)')], provides=['sideffect(sum)'])
 
         >>> # The graph works with and without 'c' provided as input.
         >>> df = pd.DataFrame({'a': [5], 'b': [2]})         # doctest: +SKIP
         >>> graph({'df': df})['sum'] == 11                  # doctest: +SKIP
         True
 
+    Note that regular data in *needs* and *provides* do not match same-named *sideffects*.
+    That is, in the following operation, the ``prices`` input is different from
+    the ``sideffect(prices)`` output:
+
+        >>> def upd_prices(sales_df, prices):
+        ...     sales_df["Prices"] = prices
+
+        >>> operation(fn=upd_prices,
+        ...           name="upd_prices",
+        ...           needs=["sales_df", "price"],
+        ...           provides=[sideffect("price")])
+        operation(name='upd_prices', needs=['sales_df', 'price'], provides=['sideffect(price)'], fn=upd_prices)
+
+    .. note::
+        An ``operation`` with *sideffects* outputs only, have functions that return
+        no value at all (like the one above).  Such operation would still be called for
+        their side-effects.
+
+    .. tip::
+        You may associate sideffects with other data to convey their relationships,
+        simply by including their names in the string - in the end, it's just a string -
+        but no enforcement will happen from *graphkit*.
+
+        >>> sideffect("price[sales_df]")
+        'sideffect(price[sales_df])'
+
     """
 
     __slots__ = ()  # avoid __dict__ on instances
 
-    def __repr__(self):
-        return "sideffect('%s')" % self
+    def __new__(cls, name):
+        return super(sideffect, cls).__new__(cls, "sideffect(%s)" % name)

test/test_graphkit.py

@@ -552,46 +552,93 @@ def addplusplus(a, b, c=0):
     assert results["sum"] == sum(named_inputs.values())
 
 
-def test_sideffects():
-    # Function without return value.
-    def extend(box):
-        box.extend([1, 2])
+# Function without return value.
+def _box_extend(box, *args):
+    box.extend([1, 2])
 
-    def increment(box):
-        for i in range(len(box)):
-            box[i] += 1
 
-    # Designate `a`, `b` as sideffect inp/out arguments.
-    graph = compose("mygraph")(
-        operation(
-            name="extend",
-            needs=["box", sideffect("a")],
-            provides=[sideffect("b")],
-        )(extend),
-        operation(
-            name="increment",
-            needs=["box", sideffect("b")],
-            provides=sideffect("c"),
-        )(increment),
-    )
+def _box_increment(box):
+    for i in range(len(box)):
+        box[i] += 1
+
 
-    assert graph({"box": [0], "a": True})["box"] == [1, 2, 3]
+@pytest.mark.parametrize("bools", range(4))
+def test_sideffect_no_real_data(bools):
+    reverse = bools >> 0 & 1
+    parallel = bools >> 1 & 1
 
-    # Reverse order of functions.
-    graph = compose("mygraph")(
+    ops = [
         operation(
-            name="increment",
-            needs=["box", sideffect("a")],
-            provides=sideffect("b"),
-        )(increment),
+            name="extend", needs=["box", sideffect("a")], provides=[sideffect("b")]
+        )(_box_extend),
         operation(
-            name="extend",
-            needs=["box", sideffect("b")],
-            provides=[sideffect("c")],
-        )(extend),
-    )
+            name="increment", needs=["box", sideffect("b")], provides=sideffect("c")
+        )(_box_increment),
+    ]
+    if reverse:
+        ops = reversed(ops)
+    # Designate `a`, `b` as sideffect inp/out arguments.
+    graph = compose("mygraph")(*ops)
+    if parallel:
+        graph.set_execution_method("parallel")
+
+    # Normal data must not match sideffects
+    with pytest.raises(ValueError, match="Unknown output node"):
+        graph({"box": [0], "a": True}, outputs=["a"])
+    with pytest.raises(ValueError, match="Unknown output node"):
+        graph({"box": [0], "a": True}, outputs=["b"])
+
+    sol = graph({"box": [0], "a": True})
+    # Nothing run if no sideffect inputs given.
+    assert not graph.net.last_plan.executed
+    assert sol == {"box": [0], "a": True}
+
+    # Nothing run if no sideffect inputs given.
+    sol = graph({"box": [0], "a": True}, outputs=["box", sideffect("b")])
+    assert not graph.net.last_plan.executed
+    assert sol == {"box": [0]}
+
+    ## OK INPUT SIDEFFECTS
+    #
+    # ok, no asked out
+    sol = graph({"box": [0], sideffect("a"): True})
+    assert sol == {"box": [1, 2, 3], sideffect("a"): True}
+    #
+    # bad, not asked the out-sideffect
+    sol = graph({"box": [0], sideffect("a"): True}, "box")
+    assert sol == {"box": [0]}
+    #
+    # ok, asked the 1st out-sideffect
+    sol = graph({"box": [0], sideffect("a"): True}, ["box", sideffect("b")])
+    assert sol == {"box": [0, 1, 2]}
+    #
+    # ok, asked the 2nd out-sideffect
+    sol = graph({"box": [0], sideffect("a"): True}, ["box", sideffect("c")])
+    assert sol == {"box": [1, 2, 3]}
+
+
+@pytest.mark.parametrize("bools", range(4))
+def test_sideffect_real_input(bools):
+    reverse = bools >> 0 & 1
+    parallel = bools >> 1 & 1
+
+    ops = [
+        operation(name="extend", needs=["box", "a"], provides=[sideffect("b")])(
+            _box_extend
+        ),
+        operation(name="increment", needs=["box", sideffect("b")], provides="c")(
+            _box_increment
+        ),
+    ]
+    if reverse:
+        ops = reversed(ops)
+    # Designate `a`, `b` as sideffect inp/out arguments.
+    graph = compose("mygraph")(*ops)
+    if parallel:
+        graph.set_execution_method("parallel")
 
-    assert graph({"box": [0], "a": None})["box"] == [1, 1, 2]
+    assert graph({"box": [0], "a": True}) == {"a": True, "box": [1, 2, 3], "c": None}
+    assert graph({"box": [0], "a": True}, ["box", "c"]) == {"box": [1, 2, 3], "c": None}
 
 
 @pytest.mark.xfail(