Proves that we can use builtin python types to fake modules
diff --git a/hamilton/ad_hoc_utils.py b/hamilton/ad_hoc_utils.py
new file mode 100644
index 0000000..0b15748
--- /dev/null
+++ b/hamilton/ad_hoc_utils.py
@@ -0,0 +1,48 @@
+"""A suite of tools for ad-hoc use"""
+import sys
+from typing import Callable
+from types import ModuleType
+import uuid
+
+import types
+
+
+def _copy_func(f):
+ """Returns a function with the same properties as the original one"""
+ fn = types.FunctionType(f.__code__, f.__globals__, f.__name__,
+ f.__defaults__, f.__closure__)
+ # in case f was given attrs (note this dict is a shallow copy):
+ fn.__dict__.update(f.__dict__)
+ fn.__annotations__ = f.__annotations__ # No idea why this is not a parameter...
+ return fn
+
+
+def _generate_unique_temp_module_name() -> str:
+ """Generates a unique module name that is a valid python variable."""
+ return f"temporary_module_{str(uuid.uuid4()).replace('-', '_')}"
+
+
+def create_temporary_module(*functions: Callable, module_name: str = None) -> ModuleType:
+ """Creates a temporary module usable by hamilton. Note this should *not* be used in production --
+ you should really be organizing your functions into modules. This is perfect in a jupyter notebook,
+ however, where you have a few functions that you want to string together to build an ad-hoc driver.
+
+ Note that this is slightly dangerous -- we want the module to look and feel like an actual module
+ so we can fully duck-type it. We thus stick it in sys.modules (checking if it already exists)
+
+ @param functions: Functions to use
+ @param module_name: Module name to use. if not provided will default to a unique one.
+ @return: The module that was created.
+ """
+ module_name = module_name if module_name is not None else _generate_unique_temp_module_name()
+ if module_name in sys.modules:
+ raise ValueError(f"Module already exists with name: {module_name}, please make it unique.")
+ module = ModuleType(module_name)
+ for fn in map(_copy_func, functions): # Just use copies so we don't mess with the original functions
+ fn_name = fn.__name__
+ if hasattr(module, fn_name):
+ raise ValueError(f"Duplicate/reserved function name: {fn_name} cannot be used to create a dynamic module.")
+ fn.__module__ = module.__name__
+ setattr(module, fn_name, fn)
+ sys.modules[module_name] = module
+ return module
diff --git a/hamilton/base.py b/hamilton/base.py
index a962192..59285a0 100644
--- a/hamilton/base.py
+++ b/hamilton/base.py
@@ -167,23 +167,3 @@
def build_result(self, **outputs: typing.Dict[str, typing.Any]) -> typing.Any:
"""Delegates to the result builder function supplied."""
return self.result_builder.build_result(**outputs)
-
-
-class TemporaryFunctionModule(object):
- """This class represents a way to encapsulate a set of functions for DAG construction.
-
- It is meant to be used in a notebook, e.g. google collab, like context where you are iterating and prototyping.
-
- Once you're happy with the functions you have created, you're supposed to transition the functions into a
- proper python module.
- """
- __name__ = 'TemporaryFunctionModule'
-
- def __init__(self, *functions: Callable):
- """Constructor.
-
- Takes in functions as arguments.
- """
- for function in functions:
- # this sets each function as an attribute on this object.
- setattr(self, function.__name__, function)
diff --git a/hamilton/driver.py b/hamilton/driver.py
index e992ecd..3dbbce9 100644
--- a/hamilton/driver.py
+++ b/hamilton/driver.py
@@ -44,7 +44,7 @@
def __init__(self,
config: Dict[str, Any],
- *modules: Union[ModuleType, base.TemporaryFunctionModule],
+ *modules: ModuleType,
adapter: base.HamiltonGraphAdapter = None):
"""Constructor: creates a DAG given the configuration & modules to crawl.
diff --git a/hamilton/graph.py b/hamilton/graph.py
index e38cbba..6b93fd2 100644
--- a/hamilton/graph.py
+++ b/hamilton/graph.py
@@ -94,7 +94,7 @@
return False
-def find_functions(function_module: Union[ModuleType, base.TemporaryFunctionModule]) -> List[Tuple[str, Callable]]:
+def find_functions(function_module: ModuleType) -> List[Tuple[str, Callable]]:
"""Function to determine the set of functions we want to build a graph from.
This iterates through the function module and grabs all function definitions.
@@ -103,8 +103,7 @@
def valid_fn(fn):
if inspect.isfunction(fn) and not fn.__name__.startswith('_'):
- return (is_submodule(inspect.getmodule(fn), function_module)
- or isinstance(function_module, base.TemporaryFunctionModule))
+ return is_submodule(inspect.getmodule(fn), function_module)
return False
return [f for f in inspect.getmembers(function_module, predicate=valid_fn)]
@@ -139,8 +138,8 @@
required_node.depended_on_by.append(func_node)
-def create_function_graph(*modules: Union[ModuleType, base.TemporaryFunctionModule],
- config: Dict[str, Any], adapter: base.HamiltonGraphAdapter) -> Dict[str, node.Node]:
+def create_function_graph(*modules: ModuleType, config: Dict[str, Any],
+ adapter: base.HamiltonGraphAdapter) -> Dict[str, node.Node]:
"""Creates a graph of all available functions & their dependencies.
:param modules: A set of modules over which one wants to compute the function graph
:param config: Dictionary that we will inspect to get values from in building the function graph.
@@ -222,7 +221,7 @@
"""
def __init__(self,
- *modules: Union[ModuleType, base.TemporaryFunctionModule],
+ *modules: ModuleType,
config: Dict[str, Any],
adapter: base.HamiltonGraphAdapter = None):
"""Initializes a function graph by crawling through modules. Function graph must have a config,
diff --git a/tests/test_graph.py b/tests/test_graph.py
index 579d046..494359a 100644
--- a/tests/test_graph.py
+++ b/tests/test_graph.py
@@ -19,7 +19,7 @@
import tests.resources.parametrized_inputs
import tests.resources.parametrized_nodes
import tests.resources.typing_vs_not_typing
-from hamilton import graph, base
+from hamilton import graph, base, ad_hoc_utils
from hamilton import node
from hamilton.node import NodeSource
@@ -39,12 +39,13 @@
expected = [('A', tests.resources.dummy_functions.A),
('B', tests.resources.dummy_functions.B),
('C', tests.resources.dummy_functions.C)]
- func_module = base.TemporaryFunctionModule(tests.resources.dummy_functions.A,
+ func_module = ad_hoc_utils.create_temporary_module(tests.resources.dummy_functions.A,
tests.resources.dummy_functions.B,
tests.resources.dummy_functions.C)
actual = graph.find_functions(func_module)
assert len(actual) == len(expected)
- assert actual == expected
+ assert [node_name for node_name, _ in actual] == [node_name for node_name, _ in expected]
+ assert [fn.__code__ for _, fn in actual] == [fn.__code__ for _, fn in expected] # easy way to say they're the same
def test_add_dependency_missing_param_type():
@@ -666,7 +667,7 @@
def my_function(A: int, b: int, c: int) -> int:
"""Function for input below"""
return A + b + c
- f_module = base.TemporaryFunctionModule(my_function)
+ f_module = ad_hoc_utils.create_temporary_module(my_function)
fg = graph.FunctionGraph(tests.resources.dummy_functions, f_module, config={'b': 3, 'c': 1})
results = fg.execute([n for n in fg.get_nodes() if n.name in ['my_function', 'A']])
assert results == {'A': 4, 'b': 3, 'c': 1, 'my_function': 8}
