Google Git
Sign in
apache/hamilton/refs/heads/fix-serialize-extract-fields/./examples/mutate/abstract_functionality_blueprint/notebook.ipynb
blob: afde97141abadfecdd76f39db2361162350e6632 [file]
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Execute this cell to install dependencies\n",
"%pip install sf-hamilton[visualization]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Using `mutate` for pre-processing and feature engineering [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dagworks-inc/hamilton/blob/main/examples/mutate/abstract_functionality_blueprint/notebook.ipynb) [![GitHub badge](https://img.shields.io/badge/github-view_source-2b3137?logo=github)](https://github.com/dagworks-inc/hamilton/blob/main/examples/mutate/abstract_functionality_blueprint/notebook.ipynb)\n",
"\n",
"\n",
"We give some application suggestions for mutating the outputs of functions in a distributed manner with `@mutate.`\n",
"When you scroll through the notebook we build examples from most straight forward applications to more complex logic that showcases the amount of flexibility you get with this decorator.\n",
"\n",
"Mutate gives the ability to apply the same transformation to the each output of multiple functions in the DAG. It can be particularly useful in the following scenarios:\n",
"\n",
"1. Loading data and applying pre-cleaning step.\n",
"2. Feature engineering via joining, filtering, sorting, applying adjustment factors on a per column basis etc.\n",
"3. Experimenting with different transformations across nodes by selectively turning transformations on / off.\n",
"\n",
"\n",
"and effectively replaces:\n",
"1. Having to have unique names and then changing wiring if you want to add/remove/replace something.\n",
"2. Enabling more verb like names on functions.\n",
"3. Potentially simpler \"reuse\" of transform functions across DAG paths..."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/jernejfrank/miniconda3/envs/hamilton/lib/python3.10/site-packages/pyspark/pandas/__init__.py:50: UserWarning: 'PYARROW_IGNORE_TIMEZONE' environment variable was not set. It is required to set this environment variable to '1' in both driver and executor sides if you use pyarrow>=2.0.0. pandas-on-Spark will set it for you but it does not work if there is a Spark context already launched.\n",
" warnings.warn(\n",
"/Users/jernejfrank/miniconda3/envs/hamilton/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
}
],
"source": [
"# from hamilton import registry\n",
"# registry.disable_autoload()\n",
"\n",
"%load_ext hamilton.plugins.jupyter_magic\n",
"from hamilton import driver"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Final data 1\n",
" col_1 col_2\n",
"3 0 d\n",
"1 2 b\n",
"0 3 a\n",
"Final data 2\n",
" col_1 col_2\n",
"col_2 \n",
"150 a 10\n",
"155 b 23\n",
"145 c 32\n",
"200 d 50\n",
"5000 e 0\n"
]
},
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 12.1.2 (20240928.0832)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"109pt\" height=\"329pt\"\n",
" viewBox=\"0.00 0.00 108.85 328.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 324.8)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-324.8 104.85,-324.8 104.85,4 -4,4\"/>\n",
"<g id=\"clust1\" class=\"cluster\">\n",
"<title>cluster__legend</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-237.8 8,-312.8 92.85,-312.8 92.85,-237.8 8,-237.8\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-295.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n",
"</g>\n",
"<!-- data_3 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>data_3</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M83.35,-63.6C83.35,-63.6 17.5,-63.6 17.5,-63.6 11.5,-63.6 5.5,-57.6 5.5,-51.6 5.5,-51.6 5.5,-12 5.5,-12 5.5,-6 11.5,0 17.5,0 17.5,0 83.35,0 83.35,0 89.35,0 95.35,-6 95.35,-12 95.35,-12 95.35,-51.6 95.35,-51.6 95.35,-57.6 89.35,-63.6 83.35,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"29.05\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_3</text>\n",
"<text text-anchor=\"start\" x=\"16.3\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>data_1</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M83.35,-145.6C83.35,-145.6 17.5,-145.6 17.5,-145.6 11.5,-145.6 5.5,-139.6 5.5,-133.6 5.5,-133.6 5.5,-94 5.5,-94 5.5,-88 11.5,-82 17.5,-82 17.5,-82 83.35,-82 83.35,-82 89.35,-82 95.35,-88 95.35,-94 95.35,-94 95.35,-133.6 95.35,-133.6 95.35,-139.6 89.35,-145.6 83.35,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"29.05\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1</text>\n",
"<text text-anchor=\"start\" x=\"16.3\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>data_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M83.35,-227.6C83.35,-227.6 17.5,-227.6 17.5,-227.6 11.5,-227.6 5.5,-221.6 5.5,-215.6 5.5,-215.6 5.5,-176 5.5,-176 5.5,-170 11.5,-164 17.5,-164 17.5,-164 83.35,-164 83.35,-164 89.35,-164 95.35,-170 95.35,-176 95.35,-176 95.35,-215.6 95.35,-215.6 95.35,-221.6 89.35,-227.6 83.35,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"29.05\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2</text>\n",
"<text text-anchor=\"start\" x=\"16.3\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- function -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>function</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-282.1C72.85,-282.1 28,-282.1 28,-282.1 22,-282.1 16,-276.1 16,-270.1 16,-270.1 16,-257.5 16,-257.5 16,-251.5 22,-245.5 28,-245.5 28,-245.5 72.85,-245.5 72.85,-245.5 78.85,-245.5 84.85,-251.5 84.85,-257.5 84.85,-257.5 84.85,-270.1 84.85,-270.1 84.85,-276.1 78.85,-282.1 72.85,-282.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-258\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x7fc68313e530>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%cell_to_module -m procedural --display\n",
"\n",
"from typing import Any, List\n",
"import pandas as pd\n",
"\n",
"\n",
"def data_1()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': [3, 2, pd.NA, 0], 'col_2': ['a', 'b', pd.NA, 'd']})\n",
" return df\n",
"\n",
"def data_2()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': ['a', 'b', pd.NA, 'd', 'e'], 'col_2': [150, 155, 145, 200, 5000]})\n",
" return df\n",
"\n",
"def data_3()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': [150, 155, 145, 200, 5000], 'col_2': [10,23, 32, 50, 0]})\n",
" return df\n",
"\n",
"# data1 and data2\n",
"def _filter(some_data:pd.DataFrame)->pd.DataFrame:\n",
" return some_data.dropna()\n",
"\n",
"# data 2\n",
"# this is for value\n",
"def _add_missing_value(some_data:pd.DataFrame, missing_row:List[Any])->pd.DataFrame:\n",
" some_data.loc[-1] = missing_row\n",
" return some_data\n",
"\n",
"# data 2\n",
"# this is for source\n",
"def _join(some_data:pd.DataFrame, other_data:pd.DataFrame)->pd.DataFrame:\n",
" return some_data.set_index('col_2').join(other_data.set_index('col_1'))\n",
"\n",
"# data1 and data2\n",
"def _sort(some_data:pd.DataFrame)->pd.DataFrame:\n",
" columns = some_data.columns\n",
" return some_data.sort_values(by=columns[0])\n",
"\n",
"if __name__ == \"__main__\":\n",
" # print(\"Filter data 1\")\n",
" # print(_filter(data_1()))\n",
" # print(\"Sort data 1\")\n",
" print(\"Final data 1\")\n",
" print(_sort(_filter(data_1())))\n",
" # print(\"Filter data 2\")\n",
" # print(_filter(data_2()))\n",
" # print(\"Add missing value data 2\")\n",
" # print(_add_missing_value(_filter(data_2()),missing_row=['c', 145]))\n",
" # print(\"Join data 2 and data 3\")\n",
" # print(_join(_add_missing_value(_filter(data_2()),missing_row=['c', 145]),other_data=data_3()))\n",
" # print(\"Sort joined dataframe\")\n",
" print(\"Final data 2\")\n",
" print(_sort(_join(_add_missing_value(_filter(data_2()),missing_row=['c', 145]),other_data=data_3())))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"node_to_execute = [\"data_1\", \"data_2\", \"feat_A\"]"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 12.1.2 (20240928.0832)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"1098pt\" height=\"302pt\"\n",
" viewBox=\"0.00 0.00 1097.70 301.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 297.8)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-297.8 1093.7,-297.8 1093.7,4 -4,4\"/>\n",
"<g id=\"clust1\" class=\"cluster\">\n",
"<title>cluster__legend</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-285.8 92.85,-285.8 92.85,-155.8 8,-155.8\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-268.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n",
"</g>\n",
"<!-- data_3 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>data_3</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M443.68,-227.6C443.68,-227.6 377.82,-227.6 377.82,-227.6 371.82,-227.6 365.82,-221.6 365.82,-215.6 365.82,-215.6 365.82,-176 365.82,-176 365.82,-170 371.82,-164 377.82,-164 377.82,-164 443.68,-164 443.68,-164 449.68,-164 455.68,-170 455.68,-176 455.68,-176 455.68,-215.6 455.68,-215.6 455.68,-221.6 449.68,-227.6 443.68,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"389.38\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_3</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_join -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>data_2.with_join</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M675.77,-145.6C675.77,-145.6 572.43,-145.6 572.43,-145.6 566.43,-145.6 560.43,-139.6 560.43,-133.6 560.43,-133.6 560.43,-94 560.43,-94 560.43,-88 566.43,-82 572.43,-82 572.43,-82 675.77,-82 675.77,-82 681.77,-82 687.77,-88 687.77,-94 687.77,-94 687.77,-133.6 687.77,-133.6 687.77,-139.6 681.77,-145.6 675.77,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"571.23\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_join</text>\n",
"<text text-anchor=\"start\" x=\"589.98\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_3&#45;&gt;data_2.with_join -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>data_3&#45;&gt;data_2.with_join</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M456.06,-180.33C477.06,-172.86 502.44,-163.61 525.05,-154.8 533.03,-151.69 541.34,-148.36 549.59,-144.99\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"550.67,-148.33 558.59,-141.3 548.01,-141.86 550.67,-148.33\"/>\n",
"</g>\n",
"<!-- data_1.with_filter_ -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>data_1.with_filter_</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M682.15,-63.6C682.15,-63.6 566.05,-63.6 566.05,-63.6 560.05,-63.6 554.05,-57.6 554.05,-51.6 554.05,-51.6 554.05,-12 554.05,-12 554.05,-6 560.05,0 566.05,0 566.05,0 682.15,0 682.15,0 688.15,0 694.15,-6 694.15,-12 694.15,-12 694.15,-51.6 694.15,-51.6 694.15,-57.6 688.15,-63.6 682.15,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"564.85\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_filter_</text>\n",
"<text text-anchor=\"start\" x=\"589.98\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_sort -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>data_1.with_sort</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M840,-63.6C840,-63.6 735.15,-63.6 735.15,-63.6 729.15,-63.6 723.15,-57.6 723.15,-51.6 723.15,-51.6 723.15,-12 723.15,-12 723.15,-6 729.15,0 735.15,0 735.15,0 840,0 840,0 846,0 852,-6 852,-12 852,-12 852,-51.6 852,-51.6 852,-57.6 846,-63.6 840,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"733.95\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_sort</text>\n",
"<text text-anchor=\"start\" x=\"753.45\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_filter_&#45;&gt;data_1.with_sort -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>data_1.with_filter_&#45;&gt;data_1.with_sort</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M694.52,-31.8C700.1,-31.8 705.76,-31.8 711.38,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"711.31,-35.3 721.31,-31.8 711.31,-28.3 711.31,-35.3\"/>\n",
"</g>\n",
"<!-- data_2.with_filter_ -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>data_2.with_filter_</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M255.45,-145.6C255.45,-145.6 139.35,-145.6 139.35,-145.6 133.35,-145.6 127.35,-139.6 127.35,-133.6 127.35,-133.6 127.35,-94 127.35,-94 127.35,-88 133.35,-82 139.35,-82 139.35,-82 255.45,-82 255.45,-82 261.45,-82 267.45,-88 267.45,-94 267.45,-94 267.45,-133.6 267.45,-133.6 267.45,-139.6 261.45,-145.6 255.45,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"138.15\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_filter_</text>\n",
"<text text-anchor=\"start\" x=\"163.27\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_add_missing_value -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>data_2.with_add_missing_value</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M513.05,-145.6C513.05,-145.6 308.45,-145.6 308.45,-145.6 302.45,-145.6 296.45,-139.6 296.45,-133.6 296.45,-133.6 296.45,-94 296.45,-94 296.45,-88 302.45,-82 308.45,-82 308.45,-82 513.05,-82 513.05,-82 519.05,-82 525.05,-88 525.05,-94 525.05,-94 525.05,-133.6 525.05,-133.6 525.05,-139.6 519.05,-145.6 513.05,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"307.25\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_add_missing_value</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_filter_&#45;&gt;data_2.with_add_missing_value -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>data_2.with_filter_&#45;&gt;data_2.with_add_missing_value</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M267.9,-113.8C273.47,-113.8 279.19,-113.8 285,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"284.72,-117.3 294.72,-113.8 284.72,-110.3 284.72,-117.3\"/>\n",
"</g>\n",
"<!-- data_1 -->\n",
"<g id=\"node10\" class=\"node\">\n",
"<title>data_1</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M958.85,-63.6C958.85,-63.6 893,-63.6 893,-63.6 887,-63.6 881,-57.6 881,-51.6 881,-51.6 881,-12 881,-12 881,-6 887,0 893,0 893,0 958.85,0 958.85,0 964.85,0 970.85,-6 970.85,-12 970.85,-12 970.85,-51.6 970.85,-51.6 970.85,-57.6 964.85,-63.6 958.85,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"904.55\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1</text>\n",
"<text text-anchor=\"start\" x=\"891.8\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_sort&#45;&gt;data_1 -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>data_1.with_sort&#45;&gt;data_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M852.36,-31.8C858.04,-31.8 863.76,-31.8 869.36,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"869.13,-35.3 879.13,-31.8 869.13,-28.3 869.13,-35.3\"/>\n",
"</g>\n",
"<!-- data_2.with_sort -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>data_2.with_sort</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M840,-145.6C840,-145.6 735.15,-145.6 735.15,-145.6 729.15,-145.6 723.15,-139.6 723.15,-133.6 723.15,-133.6 723.15,-94 723.15,-94 723.15,-88 729.15,-82 735.15,-82 735.15,-82 840,-82 840,-82 846,-82 852,-88 852,-94 852,-94 852,-133.6 852,-133.6 852,-139.6 846,-145.6 840,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"733.95\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_sort</text>\n",
"<text text-anchor=\"start\" x=\"753.45\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2 -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>data_2</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M958.85,-145.6C958.85,-145.6 893,-145.6 893,-145.6 887,-145.6 881,-139.6 881,-133.6 881,-133.6 881,-94 881,-94 881,-88 887,-82 893,-82 893,-82 958.85,-82 958.85,-82 964.85,-82 970.85,-88 970.85,-94 970.85,-94 970.85,-133.6 970.85,-133.6 970.85,-139.6 964.85,-145.6 958.85,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"904.55\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2</text>\n",
"<text text-anchor=\"start\" x=\"891.8\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_sort&#45;&gt;data_2 -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>data_2.with_sort&#45;&gt;data_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M852.36,-113.8C858.04,-113.8 863.76,-113.8 869.36,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"869.13,-117.3 879.13,-113.8 869.13,-110.3 869.13,-117.3\"/>\n",
"</g>\n",
"<!-- data_2.with_join&#45;&gt;data_2.with_sort -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>data_2.with_join&#45;&gt;data_2.with_sort</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M687.99,-113.8C695.67,-113.8 703.57,-113.8 711.4,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"711.28,-117.3 721.28,-113.8 711.28,-110.3 711.28,-117.3\"/>\n",
"</g>\n",
"<!-- data_2.with_add_missing_value&#45;&gt;data_2.with_join -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>data_2.with_add_missing_value&#45;&gt;data_2.with_join</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M525.47,-113.8C533.32,-113.8 541.11,-113.8 548.67,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"548.42,-117.3 558.42,-113.8 548.42,-110.3 548.42,-117.3\"/>\n",
"</g>\n",
"<!-- data_2_raw -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>data_2_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M86.35,-145.6C86.35,-145.6 14.5,-145.6 14.5,-145.6 8.5,-145.6 2.5,-139.6 2.5,-133.6 2.5,-133.6 2.5,-94 2.5,-94 2.5,-88 8.5,-82 14.5,-82 14.5,-82 86.35,-82 86.35,-82 92.35,-82 98.35,-88 98.35,-94 98.35,-94 98.35,-133.6 98.35,-133.6 98.35,-139.6 92.35,-145.6 86.35,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"13.3\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2_raw</text>\n",
"<text text-anchor=\"start\" x=\"16.3\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2_raw&#45;&gt;data_2.with_filter_ -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>data_2_raw&#45;&gt;data_2.with_filter_</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M98.49,-113.8C104.04,-113.8 109.81,-113.8 115.65,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"115.42,-117.3 125.42,-113.8 115.42,-110.3 115.42,-117.3\"/>\n",
"</g>\n",
"<!-- feat_A -->\n",
"<g id=\"node11\" class=\"node\">\n",
"<title>feat_A</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M1077.7,-104.6C1077.7,-104.6 1011.85,-104.6 1011.85,-104.6 1005.85,-104.6 999.85,-98.6 999.85,-92.6 999.85,-92.6 999.85,-53 999.85,-53 999.85,-47 1005.85,-41 1011.85,-41 1011.85,-41 1077.7,-41 1077.7,-41 1083.7,-41 1089.7,-47 1089.7,-53 1089.7,-53 1089.7,-92.6 1089.7,-92.6 1089.7,-98.6 1083.7,-104.6 1077.7,-104.6\"/>\n",
"<text text-anchor=\"start\" x=\"1024.15\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">feat_A</text>\n",
"<text text-anchor=\"start\" x=\"1010.65\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2&#45;&gt;feat_A -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>data_2&#45;&gt;feat_A</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M971.1,-98.3C976.92,-96.26 982.93,-94.15 988.9,-92.05\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"989.83,-95.44 998.11,-88.82 987.51,-88.83 989.83,-95.44\"/>\n",
"</g>\n",
"<!-- data_1&#45;&gt;feat_A -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>data_1&#45;&gt;feat_A</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M971.1,-47.3C976.92,-49.34 982.93,-51.45 988.9,-53.55\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"987.51,-56.77 998.11,-56.78 989.83,-50.16 987.51,-56.77\"/>\n",
"</g>\n",
"<!-- data_1_raw -->\n",
"<g id=\"node12\" class=\"node\">\n",
"<title>data_1_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M446.68,-63.6C446.68,-63.6 374.82,-63.6 374.82,-63.6 368.82,-63.6 362.82,-57.6 362.82,-51.6 362.82,-51.6 362.82,-12 362.82,-12 362.82,-6 368.82,0 374.82,0 374.82,0 446.68,0 446.68,0 452.68,0 458.68,-6 458.68,-12 458.68,-12 458.68,-51.6 458.68,-51.6 458.68,-57.6 452.68,-63.6 446.68,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"373.62\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1_raw</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1_raw&#45;&gt;data_1.with_filter_ -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>data_1_raw&#45;&gt;data_1.with_filter_</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M458.92,-31.8C483.4,-31.8 514.01,-31.8 542.14,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"542.09,-35.3 552.09,-31.8 542.09,-28.3 542.09,-35.3\"/>\n",
"</g>\n",
"<!-- function -->\n",
"<g id=\"node13\" class=\"node\">\n",
"<title>function</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n",
"</g>\n",
"<!-- output -->\n",
"<g id=\"node14\" class=\"node\">\n",
"<title>output</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M67.97,-255.1C67.97,-255.1 32.87,-255.1 32.87,-255.1 26.87,-255.1 20.87,-249.1 20.87,-243.1 20.87,-243.1 20.87,-230.5 20.87,-230.5 20.87,-224.5 26.87,-218.5 32.87,-218.5 32.87,-218.5 67.97,-218.5 67.97,-218.5 73.97,-218.5 79.97,-224.5 79.97,-230.5 79.97,-230.5 79.97,-243.1 79.97,-243.1 79.97,-249.1 73.97,-255.1 67.97,-255.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-231\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">output</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x7fc683174ac0>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>col_1</th>\n",
" <th>col_2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0</td>\n",
" <td>d</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>b</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>3</td>\n",
" <td>a</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" col_1 col_2\n",
"3 0 d\n",
"1 2 b\n",
"0 3 a"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>col_1</th>\n",
" <th>col_2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>col_2</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>150</th>\n",
" <td>a</td>\n",
" <td>10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>155</th>\n",
" <td>b</td>\n",
" <td>23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>145</th>\n",
" <td>c</td>\n",
" <td>32</td>\n",
" </tr>\n",
" <tr>\n",
" <th>200</th>\n",
" <td>d</td>\n",
" <td>50</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5000</th>\n",
" <td>e</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" col_1 col_2\n",
"col_2 \n",
"150 a 10\n",
"155 b 23\n",
"145 c 32\n",
"200 d 50\n",
"5000 e 0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>col_0</th>\n",
" <th>col_1</th>\n",
" <th>col_3</th>\n",
" <th>col_2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>d</td>\n",
" <td>0</td>\n",
" <td>200</td>\n",
" <td>50</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>b</td>\n",
" <td>2</td>\n",
" <td>155</td>\n",
" <td>23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>a</td>\n",
" <td>3</td>\n",
" <td>150</td>\n",
" <td>10</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" col_0 col_1 col_3 col_2\n",
"0 d 0 200 50\n",
"1 b 2 155 23\n",
"2 a 3 150 10"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%cell_to_module -m pipe_output --display --write_to_file --execute node_to_execute\n",
"\n",
"from typing import Any, List\n",
"import pandas as pd\n",
"from hamilton.function_modifiers import pipe_output, step, source, value, hamilton_exclude\n",
"\n",
"# data1 and data2\n",
"@hamilton_exclude\n",
"def filter_(some_data:pd.DataFrame)->pd.DataFrame:\n",
" return some_data.dropna()\n",
"\n",
"@hamilton_exclude\n",
"def test_foo(a,b,c):\n",
" return a+b+c\n",
"\n",
"# data 2\n",
"# this is for value\n",
"@hamilton_exclude\n",
"def add_missing_value(some_data:pd.DataFrame, missing_row:List[Any])->pd.DataFrame:\n",
" some_data.loc[-1] = missing_row\n",
" return some_data\n",
"\n",
"# data 2\n",
"# this is for source\n",
"@hamilton_exclude\n",
"def join(some_data:pd.DataFrame, other_data:pd.DataFrame)->pd.DataFrame:\n",
" return some_data.set_index('col_2').join(other_data.set_index('col_1'))\n",
"\n",
"# data1 and data2\n",
"@hamilton_exclude\n",
"def sort(some_data:pd.DataFrame)->pd.DataFrame:\n",
" columns = some_data.columns\n",
" return some_data.sort_values(by=columns[0])\n",
"\n",
"@pipe_output(\n",
" step(filter_),\n",
" step(sort),\n",
")\n",
"def data_1()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': [3, 2, pd.NA, 0], 'col_2': ['a', 'b', pd.NA, 'd']})\n",
" return df\n",
"\n",
"@pipe_output(\n",
" step(filter_),\n",
" step(add_missing_value,missing_row=value(['c', 145])),\n",
" step(join, other_data=source('data_3')),\n",
" step(sort),\n",
")\n",
"def data_2()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': ['a', 'b', pd.NA, 'd', 'e'], 'col_2': [150, 155, 145, 200, 5000]})\n",
" return df\n",
"\n",
"def data_3()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': [150, 155, 145, 200, 5000], 'col_2': [10,23, 32, 50, 0]})\n",
" return df\n",
"\n",
"def feat_A(data_1:pd.DataFrame, data_2:pd.DataFrame)->pd.DataFrame:\n",
" return (data_1.set_index('col_2').join(data_2.reset_index(names=['col_3']).set_index('col_1'))).reset_index(names=[\"col_0\"])\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 12.1.2 (20240928.0832)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"1098pt\" height=\"302pt\"\n",
" viewBox=\"0.00 0.00 1097.70 301.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 297.8)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-297.8 1093.7,-297.8 1093.7,4 -4,4\"/>\n",
"<g id=\"clust1\" class=\"cluster\">\n",
"<title>cluster__legend</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-285.8 92.85,-285.8 92.85,-155.8 8,-155.8\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-268.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n",
"</g>\n",
"<!-- data_3 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>data_3</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M443.68,-227.6C443.68,-227.6 377.82,-227.6 377.82,-227.6 371.82,-227.6 365.82,-221.6 365.82,-215.6 365.82,-215.6 365.82,-176 365.82,-176 365.82,-170 371.82,-164 377.82,-164 377.82,-164 443.68,-164 443.68,-164 449.68,-164 455.68,-170 455.68,-176 455.68,-176 455.68,-215.6 455.68,-215.6 455.68,-221.6 449.68,-227.6 443.68,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"389.38\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_3</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_join -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>data_2.with_join</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M675.77,-145.6C675.77,-145.6 572.43,-145.6 572.43,-145.6 566.43,-145.6 560.43,-139.6 560.43,-133.6 560.43,-133.6 560.43,-94 560.43,-94 560.43,-88 566.43,-82 572.43,-82 572.43,-82 675.77,-82 675.77,-82 681.77,-82 687.77,-88 687.77,-94 687.77,-94 687.77,-133.6 687.77,-133.6 687.77,-139.6 681.77,-145.6 675.77,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"571.23\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_join</text>\n",
"<text text-anchor=\"start\" x=\"589.98\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_3&#45;&gt;data_2.with_join -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>data_3&#45;&gt;data_2.with_join</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M456.06,-180.33C477.06,-172.86 502.44,-163.61 525.05,-154.8 533.03,-151.69 541.34,-148.36 549.59,-144.99\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"550.67,-148.33 558.59,-141.3 548.01,-141.86 550.67,-148.33\"/>\n",
"</g>\n",
"<!-- data_1.with_filter_ -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>data_1.with_filter_</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M682.15,-63.6C682.15,-63.6 566.05,-63.6 566.05,-63.6 560.05,-63.6 554.05,-57.6 554.05,-51.6 554.05,-51.6 554.05,-12 554.05,-12 554.05,-6 560.05,0 566.05,0 566.05,0 682.15,0 682.15,0 688.15,0 694.15,-6 694.15,-12 694.15,-12 694.15,-51.6 694.15,-51.6 694.15,-57.6 688.15,-63.6 682.15,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"564.85\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_filter_</text>\n",
"<text text-anchor=\"start\" x=\"589.98\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_sort -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>data_1.with_sort</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M840,-63.6C840,-63.6 735.15,-63.6 735.15,-63.6 729.15,-63.6 723.15,-57.6 723.15,-51.6 723.15,-51.6 723.15,-12 723.15,-12 723.15,-6 729.15,0 735.15,0 735.15,0 840,0 840,0 846,0 852,-6 852,-12 852,-12 852,-51.6 852,-51.6 852,-57.6 846,-63.6 840,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"733.95\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_sort</text>\n",
"<text text-anchor=\"start\" x=\"753.45\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_filter_&#45;&gt;data_1.with_sort -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>data_1.with_filter_&#45;&gt;data_1.with_sort</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M694.52,-31.8C700.1,-31.8 705.76,-31.8 711.38,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"711.31,-35.3 721.31,-31.8 711.31,-28.3 711.31,-35.3\"/>\n",
"</g>\n",
"<!-- data_2.with_filter_ -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>data_2.with_filter_</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M255.45,-145.6C255.45,-145.6 139.35,-145.6 139.35,-145.6 133.35,-145.6 127.35,-139.6 127.35,-133.6 127.35,-133.6 127.35,-94 127.35,-94 127.35,-88 133.35,-82 139.35,-82 139.35,-82 255.45,-82 255.45,-82 261.45,-82 267.45,-88 267.45,-94 267.45,-94 267.45,-133.6 267.45,-133.6 267.45,-139.6 261.45,-145.6 255.45,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"138.15\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_filter_</text>\n",
"<text text-anchor=\"start\" x=\"163.27\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_add_missing_value -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>data_2.with_add_missing_value</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M513.05,-145.6C513.05,-145.6 308.45,-145.6 308.45,-145.6 302.45,-145.6 296.45,-139.6 296.45,-133.6 296.45,-133.6 296.45,-94 296.45,-94 296.45,-88 302.45,-82 308.45,-82 308.45,-82 513.05,-82 513.05,-82 519.05,-82 525.05,-88 525.05,-94 525.05,-94 525.05,-133.6 525.05,-133.6 525.05,-139.6 519.05,-145.6 513.05,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"307.25\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_add_missing_value</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_filter_&#45;&gt;data_2.with_add_missing_value -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>data_2.with_filter_&#45;&gt;data_2.with_add_missing_value</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M267.9,-113.8C273.47,-113.8 279.19,-113.8 285,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"284.72,-117.3 294.72,-113.8 284.72,-110.3 284.72,-117.3\"/>\n",
"</g>\n",
"<!-- data_1 -->\n",
"<g id=\"node10\" class=\"node\">\n",
"<title>data_1</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M958.85,-63.6C958.85,-63.6 893,-63.6 893,-63.6 887,-63.6 881,-57.6 881,-51.6 881,-51.6 881,-12 881,-12 881,-6 887,0 893,0 893,0 958.85,0 958.85,0 964.85,0 970.85,-6 970.85,-12 970.85,-12 970.85,-51.6 970.85,-51.6 970.85,-57.6 964.85,-63.6 958.85,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"904.55\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1</text>\n",
"<text text-anchor=\"start\" x=\"891.8\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_sort&#45;&gt;data_1 -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>data_1.with_sort&#45;&gt;data_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M852.36,-31.8C858.04,-31.8 863.76,-31.8 869.36,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"869.13,-35.3 879.13,-31.8 869.13,-28.3 869.13,-35.3\"/>\n",
"</g>\n",
"<!-- data_2.with_sort -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>data_2.with_sort</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M840,-145.6C840,-145.6 735.15,-145.6 735.15,-145.6 729.15,-145.6 723.15,-139.6 723.15,-133.6 723.15,-133.6 723.15,-94 723.15,-94 723.15,-88 729.15,-82 735.15,-82 735.15,-82 840,-82 840,-82 846,-82 852,-88 852,-94 852,-94 852,-133.6 852,-133.6 852,-139.6 846,-145.6 840,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"733.95\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_sort</text>\n",
"<text text-anchor=\"start\" x=\"753.45\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2 -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>data_2</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M958.85,-145.6C958.85,-145.6 893,-145.6 893,-145.6 887,-145.6 881,-139.6 881,-133.6 881,-133.6 881,-94 881,-94 881,-88 887,-82 893,-82 893,-82 958.85,-82 958.85,-82 964.85,-82 970.85,-88 970.85,-94 970.85,-94 970.85,-133.6 970.85,-133.6 970.85,-139.6 964.85,-145.6 958.85,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"904.55\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2</text>\n",
"<text text-anchor=\"start\" x=\"891.8\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_sort&#45;&gt;data_2 -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>data_2.with_sort&#45;&gt;data_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M852.36,-113.8C858.04,-113.8 863.76,-113.8 869.36,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"869.13,-117.3 879.13,-113.8 869.13,-110.3 869.13,-117.3\"/>\n",
"</g>\n",
"<!-- data_2.with_join&#45;&gt;data_2.with_sort -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>data_2.with_join&#45;&gt;data_2.with_sort</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M687.99,-113.8C695.67,-113.8 703.57,-113.8 711.4,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"711.28,-117.3 721.28,-113.8 711.28,-110.3 711.28,-117.3\"/>\n",
"</g>\n",
"<!-- data_2.with_add_missing_value&#45;&gt;data_2.with_join -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>data_2.with_add_missing_value&#45;&gt;data_2.with_join</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M525.47,-113.8C533.32,-113.8 541.11,-113.8 548.67,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"548.42,-117.3 558.42,-113.8 548.42,-110.3 548.42,-117.3\"/>\n",
"</g>\n",
"<!-- data_2_raw -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>data_2_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M86.35,-145.6C86.35,-145.6 14.5,-145.6 14.5,-145.6 8.5,-145.6 2.5,-139.6 2.5,-133.6 2.5,-133.6 2.5,-94 2.5,-94 2.5,-88 8.5,-82 14.5,-82 14.5,-82 86.35,-82 86.35,-82 92.35,-82 98.35,-88 98.35,-94 98.35,-94 98.35,-133.6 98.35,-133.6 98.35,-139.6 92.35,-145.6 86.35,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"13.3\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2_raw</text>\n",
"<text text-anchor=\"start\" x=\"16.3\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2_raw&#45;&gt;data_2.with_filter_ -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>data_2_raw&#45;&gt;data_2.with_filter_</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M98.49,-113.8C104.04,-113.8 109.81,-113.8 115.65,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"115.42,-117.3 125.42,-113.8 115.42,-110.3 115.42,-117.3\"/>\n",
"</g>\n",
"<!-- feat_A -->\n",
"<g id=\"node11\" class=\"node\">\n",
"<title>feat_A</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M1077.7,-104.6C1077.7,-104.6 1011.85,-104.6 1011.85,-104.6 1005.85,-104.6 999.85,-98.6 999.85,-92.6 999.85,-92.6 999.85,-53 999.85,-53 999.85,-47 1005.85,-41 1011.85,-41 1011.85,-41 1077.7,-41 1077.7,-41 1083.7,-41 1089.7,-47 1089.7,-53 1089.7,-53 1089.7,-92.6 1089.7,-92.6 1089.7,-98.6 1083.7,-104.6 1077.7,-104.6\"/>\n",
"<text text-anchor=\"start\" x=\"1024.15\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">feat_A</text>\n",
"<text text-anchor=\"start\" x=\"1010.65\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2&#45;&gt;feat_A -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>data_2&#45;&gt;feat_A</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M971.1,-98.3C976.92,-96.26 982.93,-94.15 988.9,-92.05\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"989.83,-95.44 998.11,-88.82 987.51,-88.83 989.83,-95.44\"/>\n",
"</g>\n",
"<!-- data_1&#45;&gt;feat_A -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>data_1&#45;&gt;feat_A</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M971.1,-47.3C976.92,-49.34 982.93,-51.45 988.9,-53.55\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"987.51,-56.77 998.11,-56.78 989.83,-50.16 987.51,-56.77\"/>\n",
"</g>\n",
"<!-- data_1_raw -->\n",
"<g id=\"node12\" class=\"node\">\n",
"<title>data_1_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M446.68,-63.6C446.68,-63.6 374.82,-63.6 374.82,-63.6 368.82,-63.6 362.82,-57.6 362.82,-51.6 362.82,-51.6 362.82,-12 362.82,-12 362.82,-6 368.82,0 374.82,0 374.82,0 446.68,0 446.68,0 452.68,0 458.68,-6 458.68,-12 458.68,-12 458.68,-51.6 458.68,-51.6 458.68,-57.6 452.68,-63.6 446.68,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"373.62\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1_raw</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1_raw&#45;&gt;data_1.with_filter_ -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>data_1_raw&#45;&gt;data_1.with_filter_</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M458.92,-31.8C483.4,-31.8 514.01,-31.8 542.14,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"542.09,-35.3 552.09,-31.8 542.09,-28.3 542.09,-35.3\"/>\n",
"</g>\n",
"<!-- function -->\n",
"<g id=\"node13\" class=\"node\">\n",
"<title>function</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n",
"</g>\n",
"<!-- output -->\n",
"<g id=\"node14\" class=\"node\">\n",
"<title>output</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M67.97,-255.1C67.97,-255.1 32.87,-255.1 32.87,-255.1 26.87,-255.1 20.87,-249.1 20.87,-243.1 20.87,-243.1 20.87,-230.5 20.87,-230.5 20.87,-224.5 26.87,-218.5 32.87,-218.5 32.87,-218.5 67.97,-218.5 67.97,-218.5 73.97,-218.5 79.97,-224.5 79.97,-230.5 79.97,-230.5 79.97,-243.1 79.97,-243.1 79.97,-249.1 73.97,-255.1 67.97,-255.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-231\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">output</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x7fc683174c10>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>col_1</th>\n",
" <th>col_2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0</td>\n",
" <td>d</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>b</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>3</td>\n",
" <td>a</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" col_1 col_2\n",
"3 0 d\n",
"1 2 b\n",
"0 3 a"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>col_1</th>\n",
" <th>col_2</th>\n",
" </tr>\n",
" <tr>\n",
" <th>col_2</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>150</th>\n",
" <td>a</td>\n",
" <td>10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>155</th>\n",
" <td>b</td>\n",
" <td>23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>145</th>\n",
" <td>c</td>\n",
" <td>32</td>\n",
" </tr>\n",
" <tr>\n",
" <th>200</th>\n",
" <td>d</td>\n",
" <td>50</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5000</th>\n",
" <td>e</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" col_1 col_2\n",
"col_2 \n",
"150 a 10\n",
"155 b 23\n",
"145 c 32\n",
"200 d 50\n",
"5000 e 0"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>col_0</th>\n",
" <th>col_1</th>\n",
" <th>col_3</th>\n",
" <th>col_2</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>d</td>\n",
" <td>0</td>\n",
" <td>200</td>\n",
" <td>50</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>b</td>\n",
" <td>2</td>\n",
" <td>155</td>\n",
" <td>23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>a</td>\n",
" <td>3</td>\n",
" <td>150</td>\n",
" <td>10</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" col_0 col_1 col_3 col_2\n",
"0 d 0 200 50\n",
"1 b 2 155 23\n",
"2 a 3 150 10"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%cell_to_module -m mutate --display --write_to_file --execute node_to_execute\n",
"\n",
"from typing import Any, List\n",
"import pandas as pd\n",
"from hamilton.function_modifiers import mutate, apply_to, source, value\n",
"\n",
"def data_1()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': [3, 2, pd.NA, 0], 'col_2': ['a', 'b', pd.NA, 'd']})\n",
" return df\n",
"\n",
"def data_2()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': ['a', 'b', pd.NA, 'd', 'e'], 'col_2': [150, 155, 145, 200, 5000]})\n",
" return df\n",
"\n",
"def data_3()->pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({'col_1': [150, 155, 145, 200, 5000], 'col_2': [10,23, 32, 50, 0]})\n",
" return df\n",
"\n",
"# data1 and data2\n",
"@mutate(\n",
" data_1, data_2\n",
" )\n",
"def filter_(some_data:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Remove NAN values.\n",
" \n",
" Decorated with mutate this will be applied to both data_1 and data_2.\n",
" \"\"\"\n",
" return some_data.dropna()\n",
"\n",
"# data 2\n",
"# this is for value\n",
"@mutate(\n",
" data_2, \n",
" missing_row=value(['c', 145])\n",
" )\n",
"def add_missing_value(some_data:pd.DataFrame, missing_row:List[Any])->pd.DataFrame:\n",
" \"\"\"Add row to dataframe.\n",
" \n",
" The functions decorated with mutate can be viewed as steps in pipe_output in the order they\n",
" are implemented. This means that data_2 had a row removed with NAN and here we add back a row\n",
" by hand that replaces that row.\n",
" \"\"\"\n",
" some_data.loc[-1] = missing_row\n",
" return some_data\n",
"\n",
"# data 2\n",
"# this is for source\n",
"@mutate(\n",
" data_2, \n",
" other_data=source('data_3')\n",
" )\n",
"def join(some_data:pd.DataFrame, other_data:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Join two dataframes.\n",
" \n",
" We can use results from other nodes in the DAG by using the `source` functionality. Here we join\n",
" data_2 table with another table - data_3 - that is the output of another node.\n",
" \"\"\"\n",
" return some_data.set_index('col_2').join(other_data.set_index('col_1'))\n",
"\n",
"# data1 and data2\n",
"@mutate(\n",
" data_1, data_2\n",
")\n",
"def sort(some_data:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Sort dataframes by first column.\n",
" \n",
" This is the last step of our pipeline(s) and gets again applied to data_1 and data_2. We did some \n",
" light pre-processing on data_1 by removing NANs and sorting and more elaborate pre-processing on\n",
" data_2 where we added values and joined another table.\n",
" \"\"\"\n",
" columns = some_data.columns\n",
" return some_data.sort_values(by=columns[0])\n",
"\n",
"def feat_A(data_1:pd.DataFrame, data_2:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Combining two raw dataframes to create a feature.\"\"\"\n",
" return (data_1.set_index('col_2').join(data_2.reset_index(names=['col_3']).set_index('col_1'))).reset_index(names=[\"col_0\"])\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# We can run such pipelines also remotely and can track them in the UI"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from hamilton import base\n",
"from hamilton_sdk import adapters\n",
"from hamilton.plugins.h_ray import RayGraphAdapter,RayTaskExecutor\n",
"import ray\n",
"\n",
"import mutate\n",
"\n",
"remote_executor = RayTaskExecutor(num_cpus=4)\n",
"shutdown = ray.shutdown\n",
"\n",
"project_id = 3\n",
"username = \"jf\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"tracker_ray = adapters.HamiltonTracker(\n",
" project_id=project_id,\n",
" username=username,\n",
" dag_name=\"mutate ray graph adapter\",\n",
" )\n",
"ray.init()\n",
"rga = RayGraphAdapter(result_builder=base.DictResult(), shutdown_ray_on_completion=True)\n",
"dr = driver.Builder().with_modules(mutate).with_adapters(rga, tracker_ray).build()\n",
"result = dr.execute(final_vars=[\"data_1\", \"data_2\", \"feat_A\"])\n",
"print(result)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"tracker_ray = adapters.HamiltonTracker(\n",
" project_id=project_id,\n",
" username=username,\n",
" dag_name=\"mutate ray task executor\",\n",
" )\n",
"\n",
"dr = (\n",
" driver.Builder()\n",
" .enable_dynamic_execution(allow_experimental_mode=True)\n",
" .with_modules(mutate)\n",
" .with_remote_executor(remote_executor)\n",
" .with_adapters(tracker_ray)\n",
" .build()\n",
" )\n",
"\n",
"print(dr.execute(final_vars=[\"data_1\", \"data_2\", \"feat_A\"]))\n",
"if shutdown is not None:\n",
" shutdown()\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# pipe_output allows for targeting specific nodes\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 12.1.2 (20240928.0832)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"981pt\" height=\"247pt\"\n",
" viewBox=\"0.00 0.00 980.85 246.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 242.8)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-242.8 976.85,-242.8 976.85,4 -4,4\"/>\n",
"<g id=\"clust1\" class=\"cluster\">\n",
"<title>cluster__legend</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n",
"</g>\n",
"<!-- field_1.with_something_else -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>field_1.with_something_else</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M515.8,-145.6C515.8,-145.6 333.7,-145.6 333.7,-145.6 327.7,-145.6 321.7,-139.6 321.7,-133.6 321.7,-133.6 321.7,-94 321.7,-94 321.7,-88 327.7,-82 333.7,-82 333.7,-82 515.8,-82 515.8,-82 521.8,-82 527.8,-88 527.8,-94 527.8,-94 527.8,-133.6 527.8,-133.6 527.8,-139.6 521.8,-145.6 515.8,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"332.5\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1.with_something_else</text>\n",
"<text text-anchor=\"start\" x=\"417.62\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_1.transform_1 -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>field_1.transform_1</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M691.65,-145.6C691.65,-145.6 568.8,-145.6 568.8,-145.6 562.8,-145.6 556.8,-139.6 556.8,-133.6 556.8,-133.6 556.8,-94 556.8,-94 556.8,-88 562.8,-82 568.8,-82 568.8,-82 691.65,-82 691.65,-82 697.65,-82 703.65,-88 703.65,-94 703.65,-94 703.65,-133.6 703.65,-133.6 703.65,-139.6 697.65,-145.6 691.65,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"567.6\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1.transform_1</text>\n",
"<text text-anchor=\"start\" x=\"623.1\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_1.with_something_else&#45;&gt;field_1.transform_1 -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>field_1.with_something_else&#45;&gt;field_1.transform_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M528.08,-113.8C533.78,-113.8 539.48,-113.8 545.1,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"545.01,-117.3 555.01,-113.8 545.01,-110.3 545.01,-117.3\"/>\n",
"</g>\n",
"<!-- field_1.transform_2 -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>field_1.transform_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M867.5,-145.6C867.5,-145.6 744.65,-145.6 744.65,-145.6 738.65,-145.6 732.65,-139.6 732.65,-133.6 732.65,-133.6 732.65,-94 732.65,-94 732.65,-88 738.65,-82 744.65,-82 744.65,-82 867.5,-82 867.5,-82 873.5,-82 879.5,-88 879.5,-94 879.5,-94 879.5,-133.6 879.5,-133.6 879.5,-139.6 873.5,-145.6 867.5,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"743.45\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1.transform_2</text>\n",
"<text text-anchor=\"start\" x=\"798.95\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_1 -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>field_1</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M960.85,-145.6C960.85,-145.6 920.5,-145.6 920.5,-145.6 914.5,-145.6 908.5,-139.6 908.5,-133.6 908.5,-133.6 908.5,-94 908.5,-94 908.5,-88 914.5,-82 920.5,-82 920.5,-82 960.85,-82 960.85,-82 966.85,-82 972.85,-88 972.85,-94 972.85,-94 972.85,-133.6 972.85,-133.6 972.85,-139.6 966.85,-145.6 960.85,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"919.3\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1</text>\n",
"<text text-anchor=\"start\" x=\"933.55\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_1.transform_2&#45;&gt;field_1 -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>field_1.transform_2&#45;&gt;field_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M879.97,-113.8C885.68,-113.8 891.32,-113.8 896.72,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"896.59,-117.3 906.59,-113.8 896.59,-110.3 896.59,-117.3\"/>\n",
"</g>\n",
"<!-- field_3.with_something_else -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>field_3.with_something_else</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M515.8,-227.6C515.8,-227.6 333.7,-227.6 333.7,-227.6 327.7,-227.6 321.7,-221.6 321.7,-215.6 321.7,-215.6 321.7,-176 321.7,-176 321.7,-170 327.7,-164 333.7,-164 333.7,-164 515.8,-164 515.8,-164 521.8,-164 527.8,-170 527.8,-176 527.8,-176 527.8,-215.6 527.8,-215.6 527.8,-221.6 521.8,-227.6 515.8,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"332.5\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_3.with_something_else</text>\n",
"<text text-anchor=\"start\" x=\"417.62\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_3.transform_2 -->\n",
"<g id=\"node12\" class=\"node\">\n",
"<title>field_3.transform_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M691.65,-227.6C691.65,-227.6 568.8,-227.6 568.8,-227.6 562.8,-227.6 556.8,-221.6 556.8,-215.6 556.8,-215.6 556.8,-176 556.8,-176 556.8,-170 562.8,-164 568.8,-164 568.8,-164 691.65,-164 691.65,-164 697.65,-164 703.65,-170 703.65,-176 703.65,-176 703.65,-215.6 703.65,-215.6 703.65,-221.6 697.65,-227.6 691.65,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"567.6\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_3.transform_2</text>\n",
"<text text-anchor=\"start\" x=\"623.1\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_3.with_something_else&#45;&gt;field_3.transform_2 -->\n",
"<g id=\"edge12\" class=\"edge\">\n",
"<title>field_3.with_something_else&#45;&gt;field_3.transform_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M528.08,-195.8C533.78,-195.8 539.48,-195.8 545.1,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"545.01,-199.3 555.01,-195.8 545.01,-192.3 545.01,-199.3\"/>\n",
"</g>\n",
"<!-- field_3_raw -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>field_3_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M280.7,-227.6C280.7,-227.6 208.85,-227.6 208.85,-227.6 202.85,-227.6 196.85,-221.6 196.85,-215.6 196.85,-215.6 196.85,-176 196.85,-176 196.85,-170 202.85,-164 208.85,-164 208.85,-164 280.7,-164 280.7,-164 286.7,-164 292.7,-170 292.7,-176 292.7,-176 292.7,-215.6 292.7,-215.6 292.7,-221.6 286.7,-227.6 280.7,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"207.65\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_3_raw</text>\n",
"<text text-anchor=\"start\" x=\"237.65\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_3_raw&#45;&gt;field_3.with_something_else -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>field_3_raw&#45;&gt;field_3.with_something_else</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M293,-195.8C298.4,-195.8 304.06,-195.8 309.88,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"309.71,-199.3 319.71,-195.8 309.71,-192.3 309.71,-199.3\"/>\n",
"</g>\n",
"<!-- field_1.transform_1&#45;&gt;field_1.transform_2 -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>field_1.transform_1&#45;&gt;field_1.transform_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M703.95,-113.8C709.67,-113.8 715.48,-113.8 721.28,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"720.85,-117.3 730.85,-113.8 720.85,-110.3 720.85,-117.3\"/>\n",
"</g>\n",
"<!-- field_2 -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>field_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M650.4,-63.6C650.4,-63.6 610.05,-63.6 610.05,-63.6 604.05,-63.6 598.05,-57.6 598.05,-51.6 598.05,-51.6 598.05,-12 598.05,-12 598.05,-6 604.05,0 610.05,0 610.05,0 650.4,0 650.4,0 656.4,0 662.4,-6 662.4,-12 662.4,-12 662.4,-51.6 662.4,-51.6 662.4,-57.6 656.4,-63.6 650.4,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"608.85\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_2</text>\n",
"<text text-anchor=\"start\" x=\"623.1\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_1_raw -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>field_1_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M280.7,-145.6C280.7,-145.6 208.85,-145.6 208.85,-145.6 202.85,-145.6 196.85,-139.6 196.85,-133.6 196.85,-133.6 196.85,-94 196.85,-94 196.85,-88 202.85,-82 208.85,-82 208.85,-82 280.7,-82 280.7,-82 286.7,-82 292.7,-88 292.7,-94 292.7,-94 292.7,-133.6 292.7,-133.6 292.7,-139.6 286.7,-145.6 280.7,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"207.65\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1_raw</text>\n",
"<text text-anchor=\"start\" x=\"237.65\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_1_raw&#45;&gt;field_1.with_something_else -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>field_1_raw&#45;&gt;field_1.with_something_else</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M293,-113.8C298.4,-113.8 304.06,-113.8 309.88,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"309.71,-117.3 319.71,-113.8 309.71,-110.3 309.71,-117.3\"/>\n",
"</g>\n",
"<!-- field_2_raw -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>field_2_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M280.7,-63.6C280.7,-63.6 208.85,-63.6 208.85,-63.6 202.85,-63.6 196.85,-57.6 196.85,-51.6 196.85,-51.6 196.85,-12 196.85,-12 196.85,-6 202.85,0 208.85,0 208.85,0 280.7,0 280.7,0 286.7,0 292.7,-6 292.7,-12 292.7,-12 292.7,-51.6 292.7,-51.6 292.7,-57.6 286.7,-63.6 280.7,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"207.65\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_2_raw</text>\n",
"<text text-anchor=\"start\" x=\"237.65\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_2.with_something_else -->\n",
"<g id=\"node10\" class=\"node\">\n",
"<title>field_2.with_something_else</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M515.8,-63.6C515.8,-63.6 333.7,-63.6 333.7,-63.6 327.7,-63.6 321.7,-57.6 321.7,-51.6 321.7,-51.6 321.7,-12 321.7,-12 321.7,-6 327.7,0 333.7,0 333.7,0 515.8,0 515.8,0 521.8,0 527.8,-6 527.8,-12 527.8,-12 527.8,-51.6 527.8,-51.6 527.8,-57.6 521.8,-63.6 515.8,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"332.5\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_2.with_something_else</text>\n",
"<text text-anchor=\"start\" x=\"417.62\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_2_raw&#45;&gt;field_2.with_something_else -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>field_2_raw&#45;&gt;field_2.with_something_else</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M293,-31.8C298.4,-31.8 304.06,-31.8 309.88,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"309.71,-35.3 319.71,-31.8 309.71,-28.3 309.71,-35.3\"/>\n",
"</g>\n",
"<!-- field_2.with_something_else&#45;&gt;field_2 -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>field_2.with_something_else&#45;&gt;field_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M528.08,-31.8C548.67,-31.8 569.22,-31.8 586.37,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"586.14,-35.3 596.14,-31.8 586.14,-28.3 586.14,-35.3\"/>\n",
"</g>\n",
"<!-- foo -->\n",
"<g id=\"node11\" class=\"node\">\n",
"<title>foo</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-145.6C155.85,-145.6 125.85,-145.6 125.85,-145.6 119.85,-145.6 113.85,-139.6 113.85,-133.6 113.85,-133.6 113.85,-94 113.85,-94 113.85,-88 119.85,-82 125.85,-82 125.85,-82 155.85,-82 155.85,-82 161.85,-82 167.85,-88 167.85,-94 167.85,-94 167.85,-133.6 167.85,-133.6 167.85,-139.6 161.85,-145.6 155.85,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"130.35\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">foo</text>\n",
"<text text-anchor=\"start\" x=\"129.22\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Dict</text>\n",
"</g>\n",
"<!-- foo&#45;&gt;field_3_raw -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>foo&#45;&gt;field_3_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M168.1,-134.92C176.32,-141.53 185.68,-149.06 194.94,-156.51\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"192.59,-159.11 202.58,-162.66 196.98,-153.66 192.59,-159.11\"/>\n",
"</g>\n",
"<!-- foo&#45;&gt;field_1_raw -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>foo&#45;&gt;field_1_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M168.1,-113.8C173.42,-113.8 179.21,-113.8 185.15,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"185.14,-117.3 195.14,-113.8 185.14,-110.3 185.14,-117.3\"/>\n",
"</g>\n",
"<!-- foo&#45;&gt;field_2_raw -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>foo&#45;&gt;field_2_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M168.1,-92.68C176.32,-86.07 185.68,-78.54 194.94,-71.09\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"196.98,-73.94 202.58,-64.94 192.59,-68.49 196.98,-73.94\"/>\n",
"</g>\n",
"<!-- field_3 -->\n",
"<g id=\"node14\" class=\"node\">\n",
"<title>field_3</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M826.25,-227.6C826.25,-227.6 785.9,-227.6 785.9,-227.6 779.9,-227.6 773.9,-221.6 773.9,-215.6 773.9,-215.6 773.9,-176 773.9,-176 773.9,-170 779.9,-164 785.9,-164 785.9,-164 826.25,-164 826.25,-164 832.25,-164 838.25,-170 838.25,-176 838.25,-176 838.25,-215.6 838.25,-215.6 838.25,-221.6 832.25,-227.6 826.25,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"784.7\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_3</text>\n",
"<text text-anchor=\"start\" x=\"798.95\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- field_3.transform_2&#45;&gt;field_3 -->\n",
"<g id=\"edge13\" class=\"edge\">\n",
"<title>field_3.transform_2&#45;&gt;field_3</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M703.95,-195.8C723.85,-195.8 744.77,-195.8 762.39,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"762.21,-199.3 772.21,-195.8 762.21,-192.3 762.21,-199.3\"/>\n",
"</g>\n",
"<!-- a -->\n",
"<g id=\"node13\" class=\"node\">\n",
"<title>a</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.42,-145.6C65.42,-145.6 35.42,-145.6 35.42,-145.6 29.42,-145.6 23.42,-139.6 23.42,-133.6 23.42,-133.6 23.42,-94 23.42,-94 23.42,-88 29.42,-82 35.42,-82 35.42,-82 65.42,-82 65.42,-82 71.42,-82 77.42,-88 77.42,-94 77.42,-94 77.42,-133.6 77.42,-133.6 77.42,-139.6 71.42,-145.6 65.42,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"46.67\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n",
"<text text-anchor=\"start\" x=\"43.3\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- a&#45;&gt;foo -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>a&#45;&gt;foo</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M77.48,-113.8C85.16,-113.8 93.74,-113.8 102.01,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"101.86,-117.3 111.86,-113.8 101.86,-110.3 101.86,-117.3\"/>\n",
"</g>\n",
"<!-- function -->\n",
"<g id=\"node15\" class=\"node\">\n",
"<title>function</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x7fc6831ed090>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%cell_to_module -m pipe_output_on_output --display --write_to_file\n",
"from typing import Dict\n",
"from hamilton.function_modifiers import extract_fields, pipe_input, pipe_output, step, hamilton_exclude\n",
"\n",
"@hamilton_exclude\n",
"def pre_step(something:int)->int:\n",
" return something + 10\n",
"\n",
"@hamilton_exclude\n",
"def post_step(something:int)->int:\n",
" return something + 100\n",
"\n",
"@hamilton_exclude\n",
"def something_else(something:int)->int:\n",
" return something + 1000\n",
"\n",
"def a()->int:\n",
" return 10\n",
"\n",
"@pipe_output(\n",
" step(something_else), # gets applied to all sink nodes\n",
" step(pre_step).named(name=\"transform_1\").on_output(\"field_1\"), # only applied to field_1\n",
" step(post_step).named(name=\"transform_2\").on_output([\"field_1\", \"field_3\"]), # applied to field_1 and field_3\n",
")\n",
"@extract_fields(\n",
" {\"field_1\":int, \"field_2\":int, \"field_3\":int}\n",
")\n",
"def foo(a:int)->Dict[str,int]:\n",
" return {\"field_1\":1, \"field_2\":2, \"field_3\":3}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Similarly mutate allows to specify which nodes it gets applied to\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"node_to_execute = [\"data_1\", \"data_2\", \"feat_C\", \"feat_D\"]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 12.1.2 (20240928.0832)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"1534pt\" height=\"318pt\"\n",
" viewBox=\"0.00 0.00 1534.10 317.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 313.6)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-313.6 1530.1,-313.6 1530.1,4 -4,4\"/>\n",
"<g id=\"clust1\" class=\"cluster\">\n",
"<title>cluster__legend</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-196.8 8,-271.8 92.85,-271.8 92.85,-196.8 8,-196.8\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-254.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n",
"</g>\n",
"<!-- feat_D -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>feat_D</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1514.1,-268.6C1514.1,-268.6 1448.25,-268.6 1448.25,-268.6 1442.25,-268.6 1436.25,-262.6 1436.25,-256.6 1436.25,-256.6 1436.25,-217 1436.25,-217 1436.25,-211 1442.25,-205 1448.25,-205 1448.25,-205 1514.1,-205 1514.1,-205 1520.1,-205 1526.1,-211 1526.1,-217 1526.1,-217 1526.1,-256.6 1526.1,-256.6 1526.1,-262.6 1520.1,-268.6 1514.1,-268.6\"/>\n",
"<text text-anchor=\"start\" x=\"1460.55\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">feat_D</text>\n",
"<text text-anchor=\"start\" x=\"1447.05\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_sort -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>data_1.with_sort</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M795.75,-227.6C795.75,-227.6 690.9,-227.6 690.9,-227.6 684.9,-227.6 678.9,-221.6 678.9,-215.6 678.9,-215.6 678.9,-176 678.9,-176 678.9,-170 684.9,-164 690.9,-164 690.9,-164 795.75,-164 795.75,-164 801.75,-164 807.75,-170 807.75,-176 807.75,-176 807.75,-215.6 807.75,-215.6 807.75,-221.6 801.75,-227.6 795.75,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"689.7\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_sort</text>\n",
"<text text-anchor=\"start\" x=\"709.2\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1 -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>data_1</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M914.6,-227.6C914.6,-227.6 848.75,-227.6 848.75,-227.6 842.75,-227.6 836.75,-221.6 836.75,-215.6 836.75,-215.6 836.75,-176 836.75,-176 836.75,-170 842.75,-164 848.75,-164 848.75,-164 914.6,-164 914.6,-164 920.6,-164 926.6,-170 926.6,-176 926.6,-176 926.6,-215.6 926.6,-215.6 926.6,-221.6 920.6,-227.6 914.6,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"860.3\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1</text>\n",
"<text text-anchor=\"start\" x=\"847.55\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1.with_sort&#45;&gt;data_1 -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>data_1.with_sort&#45;&gt;data_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M808.11,-195.8C813.79,-195.8 819.51,-195.8 825.11,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"824.88,-199.3 834.88,-195.8 824.88,-192.3 824.88,-199.3\"/>\n",
"</g>\n",
"<!-- field_2 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>field_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1395.25,-309.6C1395.25,-309.6 1354.9,-309.6 1354.9,-309.6 1348.9,-309.6 1342.9,-303.6 1342.9,-297.6 1342.9,-297.6 1342.9,-258 1342.9,-258 1342.9,-252 1348.9,-246 1354.9,-246 1354.9,-246 1395.25,-246 1395.25,-246 1401.25,-246 1407.25,-252 1407.25,-258 1407.25,-258 1407.25,-297.6 1407.25,-297.6 1407.25,-303.6 1401.25,-309.6 1395.25,-309.6\"/>\n",
"<text text-anchor=\"start\" x=\"1353.7\" y=\"-286.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_2</text>\n",
"<text text-anchor=\"start\" x=\"1355.95\" y=\"-258.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- field_2&#45;&gt;feat_D -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>field_2&#45;&gt;feat_D</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1407.59,-265.39C1413.34,-263.12 1419.49,-260.7 1425.69,-258.26\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1426.66,-261.64 1434.68,-254.72 1424.1,-255.12 1426.66,-261.64\"/>\n",
"</g>\n",
"<!-- feat_B -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>feat_B</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1033.45,-145.6C1033.45,-145.6 967.6,-145.6 967.6,-145.6 961.6,-145.6 955.6,-139.6 955.6,-133.6 955.6,-133.6 955.6,-94 955.6,-94 955.6,-88 961.6,-82 967.6,-82 967.6,-82 1033.45,-82 1033.45,-82 1039.45,-82 1045.45,-88 1045.45,-94 1045.45,-94 1045.45,-133.6 1045.45,-133.6 1045.45,-139.6 1039.45,-145.6 1033.45,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"979.9\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">feat_B</text>\n",
"<text text-anchor=\"start\" x=\"966.4\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- col_2_raw -->\n",
"<g id=\"node10\" class=\"node\">\n",
"<title>col_2_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1154.17,-145.6C1154.17,-145.6 1090.58,-145.6 1090.58,-145.6 1084.58,-145.6 1078.58,-139.6 1078.58,-133.6 1078.58,-133.6 1078.58,-94 1078.58,-94 1078.58,-88 1084.58,-82 1090.58,-82 1090.58,-82 1154.17,-82 1154.17,-82 1160.17,-82 1166.17,-88 1166.17,-94 1166.17,-94 1166.17,-133.6 1166.17,-133.6 1166.17,-139.6 1160.17,-145.6 1154.17,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"1089.38\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">col_2_raw</text>\n",
"<text text-anchor=\"start\" x=\"1103.25\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- feat_B&#45;&gt;col_2_raw -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>feat_B&#45;&gt;col_2_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1045.81,-113.8C1052.73,-113.8 1059.95,-113.8 1067.06,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1066.95,-117.3 1076.95,-113.8 1066.95,-110.3 1066.95,-117.3\"/>\n",
"</g>\n",
"<!-- col_3_raw -->\n",
"<g id=\"node19\" class=\"node\">\n",
"<title>col_3_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1154.17,-63.6C1154.17,-63.6 1090.58,-63.6 1090.58,-63.6 1084.58,-63.6 1078.58,-57.6 1078.58,-51.6 1078.58,-51.6 1078.58,-12 1078.58,-12 1078.58,-6 1084.58,0 1090.58,0 1090.58,0 1154.17,0 1154.17,0 1160.17,0 1166.17,-6 1166.17,-12 1166.17,-12 1166.17,-51.6 1166.17,-51.6 1166.17,-57.6 1160.17,-63.6 1154.17,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"1089.38\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">col_3_raw</text>\n",
"<text text-anchor=\"start\" x=\"1103.25\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- feat_B&#45;&gt;col_3_raw -->\n",
"<g id=\"edge20\" class=\"edge\">\n",
"<title>feat_B&#45;&gt;col_3_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1045.81,-83.5C1053.4,-78.31 1061.34,-72.87 1069.11,-67.56\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1070.94,-70.55 1077.22,-62.01 1066.99,-64.77 1070.94,-70.55\"/>\n",
"</g>\n",
"<!-- data_1_raw -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>data_1_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M637.9,-227.6C637.9,-227.6 566.05,-227.6 566.05,-227.6 560.05,-227.6 554.05,-221.6 554.05,-215.6 554.05,-215.6 554.05,-176 554.05,-176 554.05,-170 560.05,-164 566.05,-164 566.05,-164 637.9,-164 637.9,-164 643.9,-164 649.9,-170 649.9,-176 649.9,-176 649.9,-215.6 649.9,-215.6 649.9,-221.6 643.9,-227.6 637.9,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"564.85\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1_raw</text>\n",
"<text text-anchor=\"start\" x=\"567.85\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_1_raw&#45;&gt;data_1.with_sort -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>data_1_raw&#45;&gt;data_1.with_sort</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M650.15,-195.8C655.73,-195.8 661.53,-195.8 667.38,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"667.13,-199.3 677.13,-195.8 667.13,-192.3 667.13,-199.3\"/>\n",
"</g>\n",
"<!-- col_2 -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>col_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1393,-145.6C1393,-145.6 1357.15,-145.6 1357.15,-145.6 1351.15,-145.6 1345.15,-139.6 1345.15,-133.6 1345.15,-133.6 1345.15,-94 1345.15,-94 1345.15,-88 1351.15,-82 1357.15,-82 1357.15,-82 1393,-82 1393,-82 1399,-82 1405,-88 1405,-94 1405,-94 1405,-133.6 1405,-133.6 1405,-139.6 1399,-145.6 1393,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"1357.82\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">col_2</text>\n",
"<text text-anchor=\"start\" x=\"1355.95\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- col_2&#45;&gt;feat_D -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>col_2&#45;&gt;feat_D</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1403.15,-145.8C1416.07,-161.05 1431.66,-179.48 1445.43,-195.75\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1442.76,-198.01 1451.89,-203.38 1448.1,-193.49 1442.76,-198.01\"/>\n",
"</g>\n",
"<!-- data_2.with_add_missing_value -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>data_2.with_add_missing_value</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M513.05,-186.6C513.05,-186.6 308.45,-186.6 308.45,-186.6 302.45,-186.6 296.45,-180.6 296.45,-174.6 296.45,-174.6 296.45,-135 296.45,-135 296.45,-129 302.45,-123 308.45,-123 308.45,-123 513.05,-123 513.05,-123 519.05,-123 525.05,-129 525.05,-135 525.05,-135 525.05,-174.6 525.05,-174.6 525.05,-180.6 519.05,-186.6 513.05,-186.6\"/>\n",
"<text text-anchor=\"start\" x=\"307.25\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_add_missing_value</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2. -->\n",
"<g id=\"node24\" class=\"node\">\n",
"<title>data_2.</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M634.9,-145.6C634.9,-145.6 569.05,-145.6 569.05,-145.6 563.05,-145.6 557.05,-139.6 557.05,-133.6 557.05,-133.6 557.05,-94 557.05,-94 557.05,-88 563.05,-82 569.05,-82 569.05,-82 634.9,-82 634.9,-82 640.9,-82 646.9,-88 646.9,-94 646.9,-94 646.9,-133.6 646.9,-133.6 646.9,-139.6 640.9,-145.6 634.9,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"578.72\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.</text>\n",
"<text text-anchor=\"start\" x=\"567.85\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_add_missing_value&#45;&gt;data_2. -->\n",
"<g id=\"edge25\" class=\"edge\">\n",
"<title>data_2.with_add_missing_value&#45;&gt;data_2.</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M525.37,-130.18C532.24,-128.69 538.97,-127.24 545.41,-125.84\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"546.13,-129.26 555.16,-123.73 544.65,-122.42 546.13,-129.26\"/>\n",
"</g>\n",
"<!-- field_1_raw -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>field_1_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1158.3,-227.6C1158.3,-227.6 1086.45,-227.6 1086.45,-227.6 1080.45,-227.6 1074.45,-221.6 1074.45,-215.6 1074.45,-215.6 1074.45,-176 1074.45,-176 1074.45,-170 1080.45,-164 1086.45,-164 1086.45,-164 1158.3,-164 1158.3,-164 1164.3,-164 1170.3,-170 1170.3,-176 1170.3,-176 1170.3,-215.6 1170.3,-215.6 1170.3,-221.6 1164.3,-227.6 1158.3,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"1085.25\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1_raw</text>\n",
"<text text-anchor=\"start\" x=\"1103.25\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- field_1.Europe -->\n",
"<g id=\"node11\" class=\"node\">\n",
"<title>field_1.Europe</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1301.9,-227.6C1301.9,-227.6 1211.3,-227.6 1211.3,-227.6 1205.3,-227.6 1199.3,-221.6 1199.3,-215.6 1199.3,-215.6 1199.3,-176 1199.3,-176 1199.3,-170 1205.3,-164 1211.3,-164 1211.3,-164 1301.9,-164 1301.9,-164 1307.9,-164 1313.9,-170 1313.9,-176 1313.9,-176 1313.9,-215.6 1313.9,-215.6 1313.9,-221.6 1307.9,-227.6 1301.9,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"1210.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1.Europe</text>\n",
"<text text-anchor=\"start\" x=\"1237.47\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- field_1_raw&#45;&gt;field_1.Europe -->\n",
"<g id=\"edge12\" class=\"edge\">\n",
"<title>field_1_raw&#45;&gt;field_1.Europe</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1170.74,-195.8C1176.34,-195.8 1182.13,-195.8 1187.94,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1187.58,-199.3 1197.58,-195.8 1187.58,-192.3 1187.58,-199.3\"/>\n",
"</g>\n",
"<!-- data_1&#45;&gt;feat_B -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>data_1&#45;&gt;feat_B</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M926.85,-164.8C933.1,-160.41 939.59,-155.86 945.99,-151.37\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"947.87,-154.32 954.05,-145.71 943.85,-148.59 947.87,-154.32\"/>\n",
"</g>\n",
"<!-- feat_A -->\n",
"<g id=\"node17\" class=\"node\">\n",
"<title>feat_A</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1019.95,-227.6C1019.95,-227.6 981.1,-227.6 981.1,-227.6 975.1,-227.6 969.1,-221.6 969.1,-215.6 969.1,-215.6 969.1,-176 969.1,-176 969.1,-170 975.1,-164 981.1,-164 981.1,-164 1019.95,-164 1019.95,-164 1025.95,-164 1031.95,-170 1031.95,-176 1031.95,-176 1031.95,-215.6 1031.95,-215.6 1031.95,-221.6 1025.95,-227.6 1019.95,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"979.9\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">feat_A</text>\n",
"<text text-anchor=\"start\" x=\"988.9\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Dict</text>\n",
"</g>\n",
"<!-- data_1&#45;&gt;feat_A -->\n",
"<g id=\"edge18\" class=\"edge\">\n",
"<title>data_1&#45;&gt;feat_A</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M926.85,-195.8C936.88,-195.8 947.52,-195.8 957.46,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"957.22,-199.3 967.22,-195.8 957.22,-192.3 957.22,-199.3\"/>\n",
"</g>\n",
"<!-- col_2.US -->\n",
"<g id=\"node25\" class=\"node\">\n",
"<title>col_2.US</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1283.9,-145.6C1283.9,-145.6 1229.3,-145.6 1229.3,-145.6 1223.3,-145.6 1217.3,-139.6 1217.3,-133.6 1217.3,-133.6 1217.3,-94 1217.3,-94 1217.3,-88 1223.3,-82 1229.3,-82 1229.3,-82 1283.9,-82 1283.9,-82 1289.9,-82 1295.9,-88 1295.9,-94 1295.9,-94 1295.9,-133.6 1295.9,-133.6 1295.9,-139.6 1289.9,-145.6 1283.9,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"1228.1\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">col_2.US</text>\n",
"<text text-anchor=\"start\" x=\"1237.47\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- col_2_raw&#45;&gt;col_2.US -->\n",
"<g id=\"edge27\" class=\"edge\">\n",
"<title>col_2_raw&#45;&gt;col_2.US</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1166.67,-113.8C1179.13,-113.8 1192.81,-113.8 1205.64,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1205.6,-117.3 1215.6,-113.8 1205.6,-110.3 1205.6,-117.3\"/>\n",
"</g>\n",
"<!-- field_1 -->\n",
"<g id=\"node13\" class=\"node\">\n",
"<title>field_1</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1395.25,-227.6C1395.25,-227.6 1354.9,-227.6 1354.9,-227.6 1348.9,-227.6 1342.9,-221.6 1342.9,-215.6 1342.9,-215.6 1342.9,-176 1342.9,-176 1342.9,-170 1348.9,-164 1354.9,-164 1354.9,-164 1395.25,-164 1395.25,-164 1401.25,-164 1407.25,-170 1407.25,-176 1407.25,-176 1407.25,-215.6 1407.25,-215.6 1407.25,-221.6 1401.25,-227.6 1395.25,-227.6\"/>\n",
"<text text-anchor=\"start\" x=\"1353.7\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_1</text>\n",
"<text text-anchor=\"start\" x=\"1355.95\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- field_1.Europe&#45;&gt;field_1 -->\n",
"<g id=\"edge14\" class=\"edge\">\n",
"<title>field_1.Europe&#45;&gt;field_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1314.13,-195.8C1319.91,-195.8 1325.72,-195.8 1331.32,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1330.93,-199.3 1340.93,-195.8 1330.93,-192.3 1330.93,-199.3\"/>\n",
"</g>\n",
"<!-- data_2.with_filter_ -->\n",
"<g id=\"node12\" class=\"node\">\n",
"<title>data_2.with_filter_</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M255.45,-186.6C255.45,-186.6 139.35,-186.6 139.35,-186.6 133.35,-186.6 127.35,-180.6 127.35,-174.6 127.35,-174.6 127.35,-135 127.35,-135 127.35,-129 133.35,-123 139.35,-123 139.35,-123 255.45,-123 255.45,-123 261.45,-123 267.45,-129 267.45,-135 267.45,-135 267.45,-174.6 267.45,-174.6 267.45,-180.6 261.45,-186.6 255.45,-186.6\"/>\n",
"<text text-anchor=\"start\" x=\"138.15\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_filter_</text>\n",
"<text text-anchor=\"start\" x=\"163.27\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_filter_&#45;&gt;data_2.with_add_missing_value -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>data_2.with_filter_&#45;&gt;data_2.with_add_missing_value</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M267.9,-154.8C273.47,-154.8 279.19,-154.8 285,-154.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"284.72,-158.3 294.72,-154.8 284.72,-151.3 284.72,-158.3\"/>\n",
"</g>\n",
"<!-- feat_C -->\n",
"<g id=\"node15\" class=\"node\">\n",
"<title>feat_C</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1514.1,-145.6C1514.1,-145.6 1448.25,-145.6 1448.25,-145.6 1442.25,-145.6 1436.25,-139.6 1436.25,-133.6 1436.25,-133.6 1436.25,-94 1436.25,-94 1436.25,-88 1442.25,-82 1448.25,-82 1448.25,-82 1514.1,-82 1514.1,-82 1520.1,-82 1526.1,-88 1526.1,-94 1526.1,-94 1526.1,-133.6 1526.1,-133.6 1526.1,-139.6 1520.1,-145.6 1514.1,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"1460.55\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">feat_C</text>\n",
"<text text-anchor=\"start\" x=\"1447.05\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- field_1&#45;&gt;feat_C -->\n",
"<g id=\"edge15\" class=\"edge\">\n",
"<title>field_1&#45;&gt;feat_C</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1407.59,-170.97C1414.9,-165.22 1422.84,-158.96 1430.71,-152.76\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1432.64,-155.7 1438.33,-146.76 1428.31,-150.2 1432.64,-155.7\"/>\n",
"</g>\n",
"<!-- data_2_raw -->\n",
"<g id=\"node14\" class=\"node\">\n",
"<title>data_2_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M86.35,-186.6C86.35,-186.6 14.5,-186.6 14.5,-186.6 8.5,-186.6 2.5,-180.6 2.5,-174.6 2.5,-174.6 2.5,-135 2.5,-135 2.5,-129 8.5,-123 14.5,-123 14.5,-123 86.35,-123 86.35,-123 92.35,-123 98.35,-129 98.35,-135 98.35,-135 98.35,-174.6 98.35,-174.6 98.35,-180.6 92.35,-186.6 86.35,-186.6\"/>\n",
"<text text-anchor=\"start\" x=\"13.3\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2_raw</text>\n",
"<text text-anchor=\"start\" x=\"16.3\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2_raw&#45;&gt;data_2.with_filter_ -->\n",
"<g id=\"edge13\" class=\"edge\">\n",
"<title>data_2_raw&#45;&gt;data_2.with_filter_</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M98.49,-154.8C104.04,-154.8 109.81,-154.8 115.65,-154.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"115.42,-158.3 125.42,-154.8 115.42,-151.3 115.42,-158.3\"/>\n",
"</g>\n",
"<!-- data_2 -->\n",
"<g id=\"node16\" class=\"node\">\n",
"<title>data_2</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M914.6,-145.6C914.6,-145.6 848.75,-145.6 848.75,-145.6 842.75,-145.6 836.75,-139.6 836.75,-133.6 836.75,-133.6 836.75,-94 836.75,-94 836.75,-88 842.75,-82 848.75,-82 848.75,-82 914.6,-82 914.6,-82 920.6,-82 926.6,-88 926.6,-94 926.6,-94 926.6,-133.6 926.6,-133.6 926.6,-139.6 920.6,-145.6 914.6,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"860.3\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2</text>\n",
"<text text-anchor=\"start\" x=\"847.55\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2&#45;&gt;feat_B -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>data_2&#45;&gt;feat_B</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M926.85,-113.8C932.52,-113.8 938.38,-113.8 944.2,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"943.77,-117.3 953.77,-113.8 943.77,-110.3 943.77,-117.3\"/>\n",
"</g>\n",
"<!-- data_2&#45;&gt;feat_A -->\n",
"<g id=\"edge19\" class=\"edge\">\n",
"<title>data_2&#45;&gt;feat_A</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M926.85,-144.8C937.54,-152.3 948.9,-160.28 959.39,-167.64\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"957.3,-170.45 967.5,-173.33 961.32,-164.72 957.3,-170.45\"/>\n",
"</g>\n",
"<!-- feat_A&#45;&gt;field_1_raw -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>feat_A&#45;&gt;field_1_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1032.39,-195.8C1041.76,-195.8 1052.38,-195.8 1062.95,-195.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1062.81,-199.3 1072.81,-195.8 1062.81,-192.3 1062.81,-199.3\"/>\n",
"</g>\n",
"<!-- field_2_raw -->\n",
"<g id=\"node23\" class=\"node\">\n",
"<title>field_2_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1158.3,-309.6C1158.3,-309.6 1086.45,-309.6 1086.45,-309.6 1080.45,-309.6 1074.45,-303.6 1074.45,-297.6 1074.45,-297.6 1074.45,-258 1074.45,-258 1074.45,-252 1080.45,-246 1086.45,-246 1086.45,-246 1158.3,-246 1158.3,-246 1164.3,-246 1170.3,-252 1170.3,-258 1170.3,-258 1170.3,-297.6 1170.3,-297.6 1170.3,-303.6 1164.3,-309.6 1158.3,-309.6\"/>\n",
"<text text-anchor=\"start\" x=\"1085.25\" y=\"-286.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_2_raw</text>\n",
"<text text-anchor=\"start\" x=\"1103.25\" y=\"-258.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- feat_A&#45;&gt;field_2_raw -->\n",
"<g id=\"edge24\" class=\"edge\">\n",
"<title>feat_A&#45;&gt;field_2_raw</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1032.39,-216.92C1042.36,-223.74 1053.75,-231.53 1064.97,-239.21\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1062.84,-241.99 1073.07,-244.75 1066.79,-236.21 1062.84,-241.99\"/>\n",
"</g>\n",
"<!-- data_3 -->\n",
"<g id=\"node18\" class=\"node\">\n",
"<title>data_3</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M443.67,-104.6C443.67,-104.6 377.82,-104.6 377.82,-104.6 371.82,-104.6 365.82,-98.6 365.82,-92.6 365.82,-92.6 365.82,-53 365.82,-53 365.82,-47 371.82,-41 377.82,-41 377.82,-41 443.67,-41 443.67,-41 449.67,-41 455.67,-47 455.67,-53 455.67,-53 455.67,-92.6 455.67,-92.6 455.67,-98.6 449.67,-104.6 443.67,-104.6\"/>\n",
"<text text-anchor=\"start\" x=\"389.37\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_3</text>\n",
"<text text-anchor=\"start\" x=\"376.62\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_3&#45;&gt;data_2. -->\n",
"<g id=\"edge26\" class=\"edge\">\n",
"<title>data_3&#45;&gt;data_2.</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M456.11,-82.41C482.83,-88.2 517.07,-95.62 545.55,-101.79\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"544.57,-105.16 555.08,-103.86 546.05,-98.32 544.57,-105.16\"/>\n",
"</g>\n",
"<!-- col_3.US -->\n",
"<g id=\"node21\" class=\"node\">\n",
"<title>col_3.US</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1283.9,-63.6C1283.9,-63.6 1229.3,-63.6 1229.3,-63.6 1223.3,-63.6 1217.3,-57.6 1217.3,-51.6 1217.3,-51.6 1217.3,-12 1217.3,-12 1217.3,-6 1223.3,0 1229.3,0 1229.3,0 1283.9,0 1283.9,0 1289.9,0 1295.9,-6 1295.9,-12 1295.9,-12 1295.9,-51.6 1295.9,-51.6 1295.9,-57.6 1289.9,-63.6 1283.9,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"1228.1\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">col_3.US</text>\n",
"<text text-anchor=\"start\" x=\"1237.47\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- col_3_raw&#45;&gt;col_3.US -->\n",
"<g id=\"edge22\" class=\"edge\">\n",
"<title>col_3_raw&#45;&gt;col_3.US</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1166.67,-31.8C1179.13,-31.8 1192.81,-31.8 1205.64,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1205.6,-35.3 1215.6,-31.8 1205.6,-28.3 1205.6,-35.3\"/>\n",
"</g>\n",
"<!-- col_3 -->\n",
"<g id=\"node20\" class=\"node\">\n",
"<title>col_3</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1393,-63.6C1393,-63.6 1357.15,-63.6 1357.15,-63.6 1351.15,-63.6 1345.15,-57.6 1345.15,-51.6 1345.15,-51.6 1345.15,-12 1345.15,-12 1345.15,-6 1351.15,0 1357.15,0 1357.15,0 1393,0 1393,0 1399,0 1405,-6 1405,-12 1405,-12 1405,-51.6 1405,-51.6 1405,-57.6 1399,-63.6 1393,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"1357.82\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">col_3</text>\n",
"<text text-anchor=\"start\" x=\"1355.95\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- col_3&#45;&gt;feat_C -->\n",
"<g id=\"edge16\" class=\"edge\">\n",
"<title>col_3&#45;&gt;feat_C</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1405.35,-54.86C1413.21,-61.05 1421.92,-67.91 1430.55,-74.71\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1428.26,-77.36 1438.28,-80.8 1432.59,-71.86 1428.26,-77.36\"/>\n",
"</g>\n",
"<!-- col_3.US&#45;&gt;col_3 -->\n",
"<g id=\"edge21\" class=\"edge\">\n",
"<title>col_3.US&#45;&gt;col_3</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1296.38,-31.8C1308.2,-31.8 1321.22,-31.8 1333.16,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1333.16,-35.3 1343.16,-31.8 1333.16,-28.3 1333.16,-35.3\"/>\n",
"</g>\n",
"<!-- data_2.with_sort -->\n",
"<g id=\"node22\" class=\"node\">\n",
"<title>data_2.with_sort</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M795.75,-145.6C795.75,-145.6 690.9,-145.6 690.9,-145.6 684.9,-145.6 678.9,-139.6 678.9,-133.6 678.9,-133.6 678.9,-94 678.9,-94 678.9,-88 684.9,-82 690.9,-82 690.9,-82 795.75,-82 795.75,-82 801.75,-82 807.75,-88 807.75,-94 807.75,-94 807.75,-133.6 807.75,-133.6 807.75,-139.6 801.75,-145.6 795.75,-145.6\"/>\n",
"<text text-anchor=\"start\" x=\"689.7\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_2.with_sort</text>\n",
"<text text-anchor=\"start\" x=\"709.2\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">DataFrame</text>\n",
"</g>\n",
"<!-- data_2.with_sort&#45;&gt;data_2 -->\n",
"<g id=\"edge17\" class=\"edge\">\n",
"<title>data_2.with_sort&#45;&gt;data_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M808.11,-113.8C813.79,-113.8 819.51,-113.8 825.11,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"824.88,-117.3 834.88,-113.8 824.88,-110.3 824.88,-117.3\"/>\n",
"</g>\n",
"<!-- field_2.Europe -->\n",
"<g id=\"node26\" class=\"node\">\n",
"<title>field_2.Europe</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M1301.9,-309.6C1301.9,-309.6 1211.3,-309.6 1211.3,-309.6 1205.3,-309.6 1199.3,-303.6 1199.3,-297.6 1199.3,-297.6 1199.3,-258 1199.3,-258 1199.3,-252 1205.3,-246 1211.3,-246 1211.3,-246 1301.9,-246 1301.9,-246 1307.9,-246 1313.9,-252 1313.9,-258 1313.9,-258 1313.9,-297.6 1313.9,-297.6 1313.9,-303.6 1307.9,-309.6 1301.9,-309.6\"/>\n",
"<text text-anchor=\"start\" x=\"1210.1\" y=\"-286.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">field_2.Europe</text>\n",
"<text text-anchor=\"start\" x=\"1237.47\" y=\"-258.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">Series</text>\n",
"</g>\n",
"<!-- field_2_raw&#45;&gt;field_2.Europe -->\n",
"<g id=\"edge28\" class=\"edge\">\n",
"<title>field_2_raw&#45;&gt;field_2.Europe</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1170.74,-277.8C1176.34,-277.8 1182.13,-277.8 1187.94,-277.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1187.58,-281.3 1197.58,-277.8 1187.58,-274.3 1187.58,-281.3\"/>\n",
"</g>\n",
"<!-- data_2.&#45;&gt;data_2.with_sort -->\n",
"<g id=\"edge23\" class=\"edge\">\n",
"<title>data_2.&#45;&gt;data_2.with_sort</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M647.07,-113.8C653.45,-113.8 660.17,-113.8 666.96,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"666.9,-117.3 676.9,-113.8 666.9,-110.3 666.9,-117.3\"/>\n",
"</g>\n",
"<!-- col_2.US&#45;&gt;col_2 -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>col_2.US&#45;&gt;col_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1296.38,-113.8C1308.2,-113.8 1321.22,-113.8 1333.16,-113.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1333.16,-117.3 1343.16,-113.8 1333.16,-110.3 1333.16,-117.3\"/>\n",
"</g>\n",
"<!-- field_2.Europe&#45;&gt;field_2 -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>field_2.Europe&#45;&gt;field_2</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M1314.13,-277.8C1319.91,-277.8 1325.72,-277.8 1331.32,-277.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"1330.93,-281.3 1340.93,-277.8 1330.93,-274.3 1330.93,-281.3\"/>\n",
"</g>\n",
"<!-- function -->\n",
"<g id=\"node27\" class=\"node\">\n",
"<title>function</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-241.1C72.85,-241.1 28,-241.1 28,-241.1 22,-241.1 16,-235.1 16,-229.1 16,-229.1 16,-216.5 16,-216.5 16,-210.5 22,-204.5 28,-204.5 28,-204.5 72.85,-204.5 72.85,-204.5 78.85,-204.5 84.85,-210.5 84.85,-216.5 84.85,-216.5 84.85,-229.1 84.85,-229.1 84.85,-235.1 78.85,-241.1 72.85,-241.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-217\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x7fc683245300>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%cell_to_module -m mutate_on_output --display --write_to_file\n",
"\n",
"from typing import Any, Dict, List\n",
"import pandas as pd\n",
"from hamilton.function_modifiers import extract_fields, extract_columns, apply_to, mutate, value, source\n",
"\n",
"\n",
"def data_1() -> pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({\"col_1\": [3, 2, pd.NA, 0], \"col_2\": [\"a\", \"b\", pd.NA, \"d\"]})\n",
" return df\n",
"\n",
"\n",
"def data_2() -> pd.DataFrame:\n",
" df = pd.DataFrame.from_dict(\n",
" {\"col_1\": [\"a\", \"b\", pd.NA, \"d\", \"e\"], \"col_2\": [150, 155, 145, 200, 5000]}\n",
" )\n",
" return df\n",
"\n",
"\n",
"def data_3() -> pd.DataFrame:\n",
" df = pd.DataFrame.from_dict({\"col_1\": [150, 155, 145, 200, 5000], \"col_2\": [10, 23, 32, 50, 0]})\n",
" return df\n",
"\n",
"\n",
"@extract_fields(\n",
" {'field_1':pd.Series,\n",
" 'field_2':pd.Series})\n",
"def feat_A(data_1:pd.DataFrame, data_2:pd.DataFrame)->Dict[str, pd.Series]:\n",
" df = (data_1.set_index('col_2').join(data_2.reset_index(names=['col_3']).set_index('col_1'))).reset_index(names=[\"col_0\"])\n",
" return {\n",
" \"field_1\": df.iloc[:, 1],\n",
" \"field_2\": df.iloc[:, 2]\n",
" }\n",
"\n",
"\n",
"@extract_columns('col_2','col_3')\n",
"def feat_B(data_1:pd.DataFrame, data_2:pd.DataFrame)->pd.DataFrame:\n",
" return (data_1.set_index('col_2').join(data_2.reset_index(names=['col_3']).set_index('col_1'))).reset_index(names=[\"col_0\"])\n",
"\n",
"\n",
"def feat_C(field_1:pd.Series, col_3:pd.Series)->pd.DataFrame:\n",
" return pd.concat([field_1, col_3], axis=1)\n",
"\n",
"def feat_D(field_2:pd.Series, col_2:pd.Series)->pd.DataFrame:\n",
" return pd.concat([field_2, col_2], axis=1)\n",
"\n",
"# data1 and data2\n",
"@mutate(\n",
" apply_to(data_1).when_in(a=[1,2,3]),\n",
" apply_to(data_2).when_not_in(a=[1,2,3])\n",
")\n",
"def filter_(some_data:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Remove NAN values.\n",
" \n",
" Mutate accepts a `config.*` family conditional where we can choose when the transform will be applied\n",
" onto the target function.\n",
" \"\"\"\n",
" return some_data.dropna()\n",
"\n",
"# data 2\n",
"# this is for value\n",
"@mutate(apply_to(data_2), missing_row=value(['c', 145]))\n",
"def add_missing_value(some_data:pd.DataFrame, missing_row:List[Any])->pd.DataFrame:\n",
" \"\"\"Add row to dataframe.\n",
" \n",
" The functions decorated with mutate can be viewed as steps in pipe_output in the order they\n",
" are implemented. This means that data_2 had a row removed with NAN and here we add back a row\n",
" by hand that replaces that row.\n",
" \"\"\"\n",
" some_data.loc[-1] = missing_row\n",
" return some_data\n",
"\n",
"# data 2\n",
"# this is for source\n",
"@mutate(apply_to(data_2).named(name=\"\",namespace=\"some_random_namespace\"), other_data=source('data_3'))\n",
"def join(some_data:pd.DataFrame, other_data:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Join two dataframes.\n",
" \n",
" We can use results from other nodes in the DAG by using the `source` functionality. Here we join\n",
" data_2 table with another table - data_3 - that is the output of another node.\n",
"\n",
" In addition, mutate also support adding custom names to the nodes.\n",
" \"\"\"\n",
" return some_data.set_index('col_2').join(other_data.set_index('col_1'))\n",
"\n",
"# data1 and data2\n",
"@mutate(\n",
" apply_to(data_1),\n",
" apply_to(data_2)\n",
")\n",
"def sort(some_data:pd.DataFrame)->pd.DataFrame:\n",
" \"\"\"Sort dataframes by first column.\n",
" \n",
" This is the last step of our pipeline(s) and gets again applied to data_1 and data_2. We did some \n",
" light pre-processing on data_1 by removing NANs and sorting and more elaborate pre-processing on\n",
" data_2 where we added values and joined another table.\n",
" \"\"\"\n",
" columns = some_data.columns\n",
" return some_data.sort_values(by=columns[0])\n",
"\n",
"\n",
"# we want to apply some adjustment coefficient to all the columns of feat_B, but only to field_1 of feat_A\n",
"@mutate(\n",
" apply_to(feat_A, factor=value(100)).on_output('field_1').named(\"Europe\"),\n",
" apply_to(feat_B, factor=value(10)).named(\"US\")\n",
")\n",
"def _adjustment_factor(some_data:pd.Series, factor:float)->pd.Series:\n",
" \"\"\"Adjust the value by some factor.\n",
" \n",
" You can imagine this step occurring later in time. We first constructed our DAG with features A and\n",
" B only to realize that something is off. We are now experimenting post-hoc to improve and find the \n",
" best possible features.\n",
"\n",
" We first split the features by columns of interest and then adjust them by a regional factor to \n",
" combine them into improved features we can use further down the pipeline.\n",
" \"\"\"\n",
" return some_data * factor\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Apply mutate twice by copying the function"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"node_to_execute = [\"data_1\"]"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 12.1.2 (20240928.0832)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"955pt\" height=\"220pt\"\n",
" viewBox=\"0.00 0.00 954.50 219.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 215.8)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-215.8 950.5,-215.8 950.5,4 -4,4\"/>\n",
"<g id=\"clust1\" class=\"cluster\">\n",
"<title>cluster__legend</title>\n",
"<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-73.8 8,-203.8 92.85,-203.8 92.85,-73.8 8,-73.8\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-186.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n",
"</g>\n",
"<!-- data_1.with_add_something_1 -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>data_1.with_add_something_1</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M841.15,-63.6C841.15,-63.6 646.3,-63.6 646.3,-63.6 640.3,-63.6 634.3,-57.6 634.3,-51.6 634.3,-51.6 634.3,-12 634.3,-12 634.3,-6 640.3,0 646.3,0 646.3,0 841.15,0 841.15,0 847.15,0 853.15,-6 853.15,-12 853.15,-12 853.15,-51.6 853.15,-51.6 853.15,-57.6 847.15,-63.6 841.15,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"645.1\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_add_something_1</text>\n",
"<text text-anchor=\"start\" x=\"736.6\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- data_1 -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>data_1</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M934.5,-63.6C934.5,-63.6 894.15,-63.6 894.15,-63.6 888.15,-63.6 882.15,-57.6 882.15,-51.6 882.15,-51.6 882.15,-12 882.15,-12 882.15,-6 888.15,0 894.15,0 894.15,0 934.5,0 934.5,0 940.5,0 946.5,-6 946.5,-12 946.5,-12 946.5,-51.6 946.5,-51.6 946.5,-57.6 940.5,-63.6 934.5,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"892.95\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1</text>\n",
"<text text-anchor=\"start\" x=\"907.2\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- data_1.with_add_something_1&#45;&gt;data_1 -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>data_1.with_add_something_1&#45;&gt;data_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M853.41,-31.8C859.3,-31.8 865.02,-31.8 870.44,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"870.25,-35.3 880.25,-31.8 870.25,-28.3 870.25,-35.3\"/>\n",
"</g>\n",
"<!-- data_1.with_add_something_more -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>data_1.with_add_something_more</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M593.3,-63.6C593.3,-63.6 372.2,-63.6 372.2,-63.6 366.2,-63.6 360.2,-57.6 360.2,-51.6 360.2,-51.6 360.2,-12 360.2,-12 360.2,-6 366.2,0 372.2,0 372.2,0 593.3,0 593.3,0 599.3,0 605.3,-6 605.3,-12 605.3,-12 605.3,-51.6 605.3,-51.6 605.3,-57.6 599.3,-63.6 593.3,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"371\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_add_something_more</text>\n",
"<text text-anchor=\"start\" x=\"475.63\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- data_1.with_add_something_more&#45;&gt;data_1.with_add_something_1 -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>data_1.with_add_something_more&#45;&gt;data_1.with_add_something_1</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M605.65,-31.8C611.26,-31.8 616.89,-31.8 622.49,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"622.44,-35.3 632.44,-31.8 622.44,-28.3 622.44,-35.3\"/>\n",
"</g>\n",
"<!-- data_1.with_add_something -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>data_1.with_add_something</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M319.2,-63.6C319.2,-63.6 139.35,-63.6 139.35,-63.6 133.35,-63.6 127.35,-57.6 127.35,-51.6 127.35,-51.6 127.35,-12 127.35,-12 127.35,-6 133.35,0 139.35,0 139.35,0 319.2,0 319.2,0 325.2,0 331.2,-6 331.2,-12 331.2,-12 331.2,-51.6 331.2,-51.6 331.2,-57.6 325.2,-63.6 319.2,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"138.15\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1.with_add_something</text>\n",
"<text text-anchor=\"start\" x=\"222.15\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- data_1.with_add_something&#45;&gt;data_1.with_add_something_more -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>data_1.with_add_something&#45;&gt;data_1.with_add_something_more</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M331.47,-31.8C337.21,-31.8 343.02,-31.8 348.85,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"348.56,-35.3 358.56,-31.8 348.56,-28.3 348.56,-35.3\"/>\n",
"</g>\n",
"<!-- data_1_raw -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>data_1_raw</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M86.35,-63.6C86.35,-63.6 14.5,-63.6 14.5,-63.6 8.5,-63.6 2.5,-57.6 2.5,-51.6 2.5,-51.6 2.5,-12 2.5,-12 2.5,-6 8.5,0 14.5,0 14.5,0 86.35,0 86.35,0 92.35,0 98.35,-6 98.35,-12 98.35,-12 98.35,-51.6 98.35,-51.6 98.35,-57.6 92.35,-63.6 86.35,-63.6\"/>\n",
"<text text-anchor=\"start\" x=\"13.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">data_1_raw</text>\n",
"<text text-anchor=\"start\" x=\"43.3\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">int</text>\n",
"</g>\n",
"<!-- data_1_raw&#45;&gt;data_1.with_add_something -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>data_1_raw&#45;&gt;data_1.with_add_something</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M98.82,-31.8C104.24,-31.8 109.93,-31.8 115.77,-31.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"115.65,-35.3 125.65,-31.8 115.65,-28.3 115.65,-35.3\"/>\n",
"</g>\n",
"<!-- function -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>function</title>\n",
"<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-118.1C72.85,-118.1 28,-118.1 28,-118.1 22,-118.1 16,-112.1 16,-106.1 16,-106.1 16,-93.5 16,-93.5 16,-87.5 22,-81.5 28,-81.5 28,-81.5 72.85,-81.5 72.85,-81.5 78.85,-81.5 84.85,-87.5 84.85,-93.5 84.85,-93.5 84.85,-106.1 84.85,-106.1 84.85,-112.1 78.85,-118.1 72.85,-118.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-94\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n",
"</g>\n",
"<!-- output -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>output</title>\n",
"<path fill=\"#ffc857\" stroke=\"black\" d=\"M67.97,-173.1C67.97,-173.1 32.87,-173.1 32.87,-173.1 26.87,-173.1 20.87,-167.1 20.87,-161.1 20.87,-161.1 20.87,-148.5 20.87,-148.5 20.87,-142.5 26.87,-136.5 32.87,-136.5 32.87,-136.5 67.97,-136.5 67.97,-136.5 73.97,-136.5 79.97,-142.5 79.97,-148.5 79.97,-148.5 79.97,-161.1 79.97,-161.1 79.97,-167.1 73.97,-173.1 67.97,-173.1\"/>\n",
"<text text-anchor=\"middle\" x=\"50.42\" y=\"-149\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">output</text>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x7fc6832447c0>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"1210"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"%%cell_to_module -m mutate_twice_the_same --display --write_to_file --execute node_to_execute\n",
"\n",
"from typing import Any, List\n",
"import pandas as pd\n",
"from hamilton.function_modifiers import mutate, apply_to, source, value\n",
"\n",
"\n",
"def data_1()->int:\n",
" return 10\n",
"\n",
"\n",
"@mutate(data_1)\n",
"def add_something(user_input:int)->int:\n",
" return user_input + 100\n",
"\n",
"@mutate(data_1)\n",
"def add_something_more(user_input:int)->int:\n",
" return user_input + 1000\n",
"\n",
"@mutate(data_1)\n",
"def add_something(user_input:int)->int: # noqa\n",
" return user_input + 100\n",
"\n",
"\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "hamilton",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.14"
}
},
"nbformat": 4,
"nbformat_minor": 2
}