| { |
| "cells": [ |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "id": "initial_id", |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "# Execute this cell to install dependencies\n", |
| "%pip install sf-hamilton[visualization]" |
| ] |
| }, |
| { |
| "metadata": {}, |
| "cell_type": "markdown", |
| "source": "# run me in google colab [](https://colab.research.google.com/github/dagworks-inc/hamilton/blob/main/examples/parallelism/lazy_threadpool_execution/notebook.ipynb) [](https://github.com/dagworks-inc/hamilton/blob/main/examples/parallelism/lazy_threadpool_execution/notebook.ipynb)\n", |
| "id": "7b55978b426b6e42" |
| }, |
| { |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:00:01.808537Z", |
| "start_time": "2025-01-02T04:59:53.847872Z" |
| } |
| }, |
| "cell_type": "code", |
| "source": "%load_ext hamilton.plugins.jupyter_magic", |
| "id": "a1f2f8937a0b0488", |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "/Users/stefankrawczyk/.pyenv/versions/knowledge_retrieval-py39/lib/python3.9/site-packages/pyspark/pandas/__init__.py:50: UserWarning: 'PYARROW_IGNORE_TIMEZONE' environment variable was not set. It is required to set this environment variable to '1' in both driver and executor sides if you use pyarrow>=2.0.0. pandas-on-Spark will set it for you but it does not work if there is a Spark context already launched.\n", |
| " warnings.warn(\n" |
| ] |
| } |
| ], |
| "execution_count": 1 |
| }, |
| { |
| "metadata": {}, |
| "cell_type": "markdown", |
| "source": [ |
| "# Create a module with some functions\n", |
| "This hopefully shows a good example of what could be parallelized given the structure of the DAG." |
| ], |
| "id": "32c01561386fc348" |
| }, |
| { |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:00:16.514305Z", |
| "start_time": "2025-01-02T05:00:16.101379Z" |
| } |
| }, |
| "cell_type": "code", |
| "source": [ |
| "%%cell_to_module my_functions --display\n", |
| "\n", |
| "import time\n", |
| "\n", |
| "\n", |
| "def a() -> str:\n", |
| " print(\"a\")\n", |
| " time.sleep(3)\n", |
| " return \"a\"\n", |
| "\n", |
| "\n", |
| "def b() -> str:\n", |
| " print(\"b\")\n", |
| " time.sleep(3)\n", |
| " return \"b\"\n", |
| "\n", |
| "\n", |
| "def c(a: str, b: str) -> str:\n", |
| " print(\"c\")\n", |
| " time.sleep(3)\n", |
| " return a + \" \" + b\n", |
| "\n", |
| "\n", |
| "def d() -> str:\n", |
| " print(\"d\")\n", |
| " time.sleep(3)\n", |
| " return \"d\"\n", |
| "\n", |
| "\n", |
| "def e(c: str, d: str) -> str:\n", |
| " print(\"e\")\n", |
| " time.sleep(3)\n", |
| " return c + \" \" + d\n", |
| "\n", |
| "\n", |
| "def z() -> str:\n", |
| " print(\"z\")\n", |
| " time.sleep(3)\n", |
| " return \"z\"\n", |
| "\n", |
| "\n", |
| "def y() -> str:\n", |
| " print(\"y\")\n", |
| " time.sleep(3)\n", |
| " return \"y\"\n", |
| "\n", |
| "\n", |
| "def x(z: str, y: str) -> str:\n", |
| " print(\"x\")\n", |
| " time.sleep(3)\n", |
| " return z + \" \" + y\n", |
| "\n", |
| "\n", |
| "def s(x: str, e: str) -> str:\n", |
| " print(\"s\")\n", |
| " time.sleep(3)\n", |
| " return x + \" \" + e\n", |
| "\n" |
| ], |
| "id": "8e0e3b7a96ca1d44", |
| "outputs": [ |
| { |
| "data": { |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n -->\n<!-- Pages: 1 -->\n<svg width=\"342pt\" height=\"359pt\"\n viewBox=\"0.00 0.00 341.85 358.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 354.6)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-354.6 337.85,-354.6 337.85,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster__legend</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n</g>\n<!-- c -->\n<g id=\"node1\" class=\"node\">\n<title>c</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-104.6C155.85,-104.6 125.85,-104.6 125.85,-104.6 119.85,-104.6 113.85,-98.6 113.85,-92.6 113.85,-92.6 113.85,-53 113.85,-53 113.85,-47 119.85,-41 125.85,-41 125.85,-41 155.85,-41 155.85,-41 161.85,-41 167.85,-47 167.85,-53 167.85,-53 167.85,-92.6 167.85,-92.6 167.85,-98.6 161.85,-104.6 155.85,-104.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">c</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- e -->\n<g id=\"node4\" class=\"node\">\n<title>e</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-186.6C238.85,-186.6 208.85,-186.6 208.85,-186.6 202.85,-186.6 196.85,-180.6 196.85,-174.6 196.85,-174.6 196.85,-135 196.85,-135 196.85,-129 202.85,-123 208.85,-123 208.85,-123 238.85,-123 238.85,-123 244.85,-123 250.85,-129 250.85,-135 250.85,-135 250.85,-174.6 250.85,-174.6 250.85,-180.6 244.85,-186.6 238.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">e</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- c->e -->\n<g id=\"edge5\" class=\"edge\">\n<title>c->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-99.2C174.51,-105.86 181.67,-113.11 188.59,-120.12\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.97,-122.44 195.48,-127.09 190.95,-117.52 185.97,-122.44\"/>\n</g>\n<!-- x -->\n<g id=\"node2\" class=\"node\">\n<title>x</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-268.6C238.85,-268.6 208.85,-268.6 208.85,-268.6 202.85,-268.6 196.85,-262.6 196.85,-256.6 196.85,-256.6 196.85,-217 196.85,-217 196.85,-211 202.85,-205 208.85,-205 208.85,-205 238.85,-205 238.85,-205 244.85,-205 250.85,-211 250.85,-217 250.85,-217 250.85,-256.6 250.85,-256.6 250.85,-262.6 244.85,-268.6 238.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">x</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- s -->\n<g id=\"node6\" class=\"node\">\n<title>s</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M321.85,-227.6C321.85,-227.6 291.85,-227.6 291.85,-227.6 285.85,-227.6 279.85,-221.6 279.85,-215.6 279.85,-215.6 279.85,-176 279.85,-176 279.85,-170 285.85,-164 291.85,-164 291.85,-164 321.85,-164 321.85,-164 327.85,-164 333.85,-170 333.85,-176 333.85,-176 333.85,-215.6 333.85,-215.6 333.85,-221.6 327.85,-227.6 321.85,-227.6\"/>\n<text text-anchor=\"start\" x=\"303.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">s</text>\n<text text-anchor=\"start\" x=\"299.35\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- x->s -->\n<g id=\"edge7\" class=\"edge\">\n<title>x->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-223.6C256.79,-220.63 263.12,-217.43 269.33,-214.29\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"270.85,-217.44 278.2,-209.8 267.69,-211.19 270.85,-217.44\"/>\n</g>\n<!-- a -->\n<g id=\"node3\" class=\"node\">\n<title>a</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-145.6C65.43,-145.6 35.43,-145.6 35.43,-145.6 29.43,-145.6 23.43,-139.6 23.43,-133.6 23.43,-133.6 23.43,-94 23.43,-94 23.43,-88 29.43,-82 35.43,-82 35.43,-82 65.43,-82 65.43,-82 71.43,-82 77.43,-88 77.43,-94 77.43,-94 77.43,-133.6 77.43,-133.6 77.43,-139.6 71.43,-145.6 65.43,-145.6\"/>\n<text text-anchor=\"start\" x=\"46.68\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- a->c -->\n<g id=\"edge1\" class=\"edge\">\n<title>a->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-101.72C85.52,-97.99 94.53,-93.81 103.15,-89.82\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"104.4,-93.1 112,-85.71 101.45,-86.75 104.4,-93.1\"/>\n</g>\n<!-- e->s -->\n<g id=\"edge8\" class=\"edge\">\n<title>e->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-168C256.79,-170.97 263.12,-174.17 269.33,-177.31\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.69,-180.41 278.2,-181.8 270.85,-174.16 267.69,-180.41\"/>\n</g>\n<!-- b -->\n<g id=\"node5\" class=\"node\">\n<title>b</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-63.6C65.43,-63.6 35.43,-63.6 35.43,-63.6 29.43,-63.6 23.43,-57.6 23.43,-51.6 23.43,-51.6 23.43,-12 23.43,-12 23.43,-6 29.43,0 35.43,0 35.43,0 65.43,0 65.43,0 71.43,0 77.43,-6 77.43,-12 77.43,-12 77.43,-51.6 77.43,-51.6 77.43,-57.6 71.43,-63.6 65.43,-63.6\"/>\n<text text-anchor=\"start\" x=\"46.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">b</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- b->c -->\n<g id=\"edge2\" class=\"edge\">\n<title>b->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-43.88C85.52,-47.61 94.53,-51.79 103.15,-55.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"101.45,-58.85 112,-59.89 104.4,-52.5 101.45,-58.85\"/>\n</g>\n<!-- z -->\n<g id=\"node7\" class=\"node\">\n<title>z</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-350.6C155.85,-350.6 125.85,-350.6 125.85,-350.6 119.85,-350.6 113.85,-344.6 113.85,-338.6 113.85,-338.6 113.85,-299 113.85,-299 113.85,-293 119.85,-287 125.85,-287 125.85,-287 155.85,-287 155.85,-287 161.85,-287 167.85,-293 167.85,-299 167.85,-299 167.85,-338.6 167.85,-338.6 167.85,-344.6 161.85,-350.6 155.85,-350.6\"/>\n<text text-anchor=\"start\" x=\"137.48\" y=\"-327.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">z</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-299.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- z->x -->\n<g id=\"edge3\" class=\"edge\">\n<title>z->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-292.4C174.51,-285.74 181.67,-278.49 188.59,-271.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"190.95,-274.08 195.48,-264.51 185.97,-269.16 190.95,-274.08\"/>\n</g>\n<!-- d -->\n<g id=\"node8\" class=\"node\">\n<title>d</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-186.6C155.85,-186.6 125.85,-186.6 125.85,-186.6 119.85,-186.6 113.85,-180.6 113.85,-174.6 113.85,-174.6 113.85,-135 113.85,-135 113.85,-129 119.85,-123 125.85,-123 125.85,-123 155.85,-123 155.85,-123 161.85,-123 167.85,-129 167.85,-135 167.85,-135 167.85,-174.6 167.85,-174.6 167.85,-180.6 161.85,-186.6 155.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"136.73\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">d</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- d->e -->\n<g id=\"edge6\" class=\"edge\">\n<title>d->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-154.8C173.5,-154.8 179.51,-154.8 185.42,-154.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-158.3 195.03,-154.8 185.03,-151.3 185.03,-158.3\"/>\n</g>\n<!-- y -->\n<g id=\"node9\" class=\"node\">\n<title>y</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-268.6C155.85,-268.6 125.85,-268.6 125.85,-268.6 119.85,-268.6 113.85,-262.6 113.85,-256.6 113.85,-256.6 113.85,-217 113.85,-217 113.85,-211 119.85,-205 125.85,-205 125.85,-205 155.85,-205 155.85,-205 161.85,-205 167.85,-211 167.85,-217 167.85,-217 167.85,-256.6 167.85,-256.6 167.85,-262.6 161.85,-268.6 155.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">y</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- y->x -->\n<g id=\"edge4\" class=\"edge\">\n<title>y->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-236.8C173.5,-236.8 179.51,-236.8 185.42,-236.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-240.3 195.03,-236.8 185.03,-233.3 185.03,-240.3\"/>\n</g>\n<!-- function -->\n<g id=\"node10\" class=\"node\">\n<title>function</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n</g>\n</g>\n</svg>\n", |
| "text/plain": [ |
| "<graphviz.graphs.Digraph at 0x14a889520>" |
| ] |
| }, |
| "metadata": {}, |
| "output_type": "display_data" |
| } |
| ], |
| "execution_count": 2 |
| }, |
| { |
| "metadata": {}, |
| "cell_type": "markdown", |
| "source": "# Run it without the adapter", |
| "id": "60355598274f9b79" |
| }, |
| { |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:02:44.719265Z", |
| "start_time": "2025-01-02T05:02:44.423066Z" |
| } |
| }, |
| "cell_type": "code", |
| "source": [ |
| "from hamilton import driver\n", |
| "dr = driver.Builder().with_modules(my_functions).build()\n", |
| "dr" |
| ], |
| "id": "fcb0677daf5b4a31", |
| "outputs": [ |
| { |
| "data": { |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n -->\n<!-- Pages: 1 -->\n<svg width=\"342pt\" height=\"359pt\"\n viewBox=\"0.00 0.00 341.85 358.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 354.6)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-354.6 337.85,-354.6 337.85,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster__legend</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n</g>\n<!-- c -->\n<g id=\"node1\" class=\"node\">\n<title>c</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-104.6C155.85,-104.6 125.85,-104.6 125.85,-104.6 119.85,-104.6 113.85,-98.6 113.85,-92.6 113.85,-92.6 113.85,-53 113.85,-53 113.85,-47 119.85,-41 125.85,-41 125.85,-41 155.85,-41 155.85,-41 161.85,-41 167.85,-47 167.85,-53 167.85,-53 167.85,-92.6 167.85,-92.6 167.85,-98.6 161.85,-104.6 155.85,-104.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">c</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- e -->\n<g id=\"node4\" class=\"node\">\n<title>e</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-186.6C238.85,-186.6 208.85,-186.6 208.85,-186.6 202.85,-186.6 196.85,-180.6 196.85,-174.6 196.85,-174.6 196.85,-135 196.85,-135 196.85,-129 202.85,-123 208.85,-123 208.85,-123 238.85,-123 238.85,-123 244.85,-123 250.85,-129 250.85,-135 250.85,-135 250.85,-174.6 250.85,-174.6 250.85,-180.6 244.85,-186.6 238.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">e</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- c->e -->\n<g id=\"edge5\" class=\"edge\">\n<title>c->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-99.2C174.51,-105.86 181.67,-113.11 188.59,-120.12\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.97,-122.44 195.48,-127.09 190.95,-117.52 185.97,-122.44\"/>\n</g>\n<!-- x -->\n<g id=\"node2\" class=\"node\">\n<title>x</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-268.6C238.85,-268.6 208.85,-268.6 208.85,-268.6 202.85,-268.6 196.85,-262.6 196.85,-256.6 196.85,-256.6 196.85,-217 196.85,-217 196.85,-211 202.85,-205 208.85,-205 208.85,-205 238.85,-205 238.85,-205 244.85,-205 250.85,-211 250.85,-217 250.85,-217 250.85,-256.6 250.85,-256.6 250.85,-262.6 244.85,-268.6 238.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">x</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- s -->\n<g id=\"node6\" class=\"node\">\n<title>s</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M321.85,-227.6C321.85,-227.6 291.85,-227.6 291.85,-227.6 285.85,-227.6 279.85,-221.6 279.85,-215.6 279.85,-215.6 279.85,-176 279.85,-176 279.85,-170 285.85,-164 291.85,-164 291.85,-164 321.85,-164 321.85,-164 327.85,-164 333.85,-170 333.85,-176 333.85,-176 333.85,-215.6 333.85,-215.6 333.85,-221.6 327.85,-227.6 321.85,-227.6\"/>\n<text text-anchor=\"start\" x=\"303.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">s</text>\n<text text-anchor=\"start\" x=\"299.35\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- x->s -->\n<g id=\"edge7\" class=\"edge\">\n<title>x->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-223.6C256.79,-220.63 263.12,-217.43 269.33,-214.29\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"270.85,-217.44 278.2,-209.8 267.69,-211.19 270.85,-217.44\"/>\n</g>\n<!-- a -->\n<g id=\"node3\" class=\"node\">\n<title>a</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-145.6C65.43,-145.6 35.43,-145.6 35.43,-145.6 29.43,-145.6 23.43,-139.6 23.43,-133.6 23.43,-133.6 23.43,-94 23.43,-94 23.43,-88 29.43,-82 35.43,-82 35.43,-82 65.43,-82 65.43,-82 71.43,-82 77.43,-88 77.43,-94 77.43,-94 77.43,-133.6 77.43,-133.6 77.43,-139.6 71.43,-145.6 65.43,-145.6\"/>\n<text text-anchor=\"start\" x=\"46.68\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- a->c -->\n<g id=\"edge1\" class=\"edge\">\n<title>a->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-101.72C85.52,-97.99 94.53,-93.81 103.15,-89.82\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"104.4,-93.1 112,-85.71 101.45,-86.75 104.4,-93.1\"/>\n</g>\n<!-- e->s -->\n<g id=\"edge8\" class=\"edge\">\n<title>e->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-168C256.79,-170.97 263.12,-174.17 269.33,-177.31\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.69,-180.41 278.2,-181.8 270.85,-174.16 267.69,-180.41\"/>\n</g>\n<!-- b -->\n<g id=\"node5\" class=\"node\">\n<title>b</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-63.6C65.43,-63.6 35.43,-63.6 35.43,-63.6 29.43,-63.6 23.43,-57.6 23.43,-51.6 23.43,-51.6 23.43,-12 23.43,-12 23.43,-6 29.43,0 35.43,0 35.43,0 65.43,0 65.43,0 71.43,0 77.43,-6 77.43,-12 77.43,-12 77.43,-51.6 77.43,-51.6 77.43,-57.6 71.43,-63.6 65.43,-63.6\"/>\n<text text-anchor=\"start\" x=\"46.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">b</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- b->c -->\n<g id=\"edge2\" class=\"edge\">\n<title>b->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-43.88C85.52,-47.61 94.53,-51.79 103.15,-55.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"101.45,-58.85 112,-59.89 104.4,-52.5 101.45,-58.85\"/>\n</g>\n<!-- z -->\n<g id=\"node7\" class=\"node\">\n<title>z</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-350.6C155.85,-350.6 125.85,-350.6 125.85,-350.6 119.85,-350.6 113.85,-344.6 113.85,-338.6 113.85,-338.6 113.85,-299 113.85,-299 113.85,-293 119.85,-287 125.85,-287 125.85,-287 155.85,-287 155.85,-287 161.85,-287 167.85,-293 167.85,-299 167.85,-299 167.85,-338.6 167.85,-338.6 167.85,-344.6 161.85,-350.6 155.85,-350.6\"/>\n<text text-anchor=\"start\" x=\"137.48\" y=\"-327.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">z</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-299.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- z->x -->\n<g id=\"edge3\" class=\"edge\">\n<title>z->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-292.4C174.51,-285.74 181.67,-278.49 188.59,-271.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"190.95,-274.08 195.48,-264.51 185.97,-269.16 190.95,-274.08\"/>\n</g>\n<!-- d -->\n<g id=\"node8\" class=\"node\">\n<title>d</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-186.6C155.85,-186.6 125.85,-186.6 125.85,-186.6 119.85,-186.6 113.85,-180.6 113.85,-174.6 113.85,-174.6 113.85,-135 113.85,-135 113.85,-129 119.85,-123 125.85,-123 125.85,-123 155.85,-123 155.85,-123 161.85,-123 167.85,-129 167.85,-135 167.85,-135 167.85,-174.6 167.85,-174.6 167.85,-180.6 161.85,-186.6 155.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"136.73\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">d</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- d->e -->\n<g id=\"edge6\" class=\"edge\">\n<title>d->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-154.8C173.5,-154.8 179.51,-154.8 185.42,-154.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-158.3 195.03,-154.8 185.03,-151.3 185.03,-158.3\"/>\n</g>\n<!-- y -->\n<g id=\"node9\" class=\"node\">\n<title>y</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-268.6C155.85,-268.6 125.85,-268.6 125.85,-268.6 119.85,-268.6 113.85,-262.6 113.85,-256.6 113.85,-256.6 113.85,-217 113.85,-217 113.85,-211 119.85,-205 125.85,-205 125.85,-205 155.85,-205 155.85,-205 161.85,-205 167.85,-211 167.85,-217 167.85,-217 167.85,-256.6 167.85,-256.6 167.85,-262.6 161.85,-268.6 155.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">y</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- y->x -->\n<g id=\"edge4\" class=\"edge\">\n<title>y->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-236.8C173.5,-236.8 179.51,-236.8 185.42,-236.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-240.3 195.03,-236.8 185.03,-233.3 185.03,-240.3\"/>\n</g>\n<!-- function -->\n<g id=\"node10\" class=\"node\">\n<title>function</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n</g>\n</g>\n</svg>\n", |
| "text/plain": [ |
| "<hamilton.driver.Driver at 0x1508cca90>" |
| ] |
| }, |
| "execution_count": 6, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "execution_count": 6 |
| }, |
| { |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:03:18.620774Z", |
| "start_time": "2025-01-02T05:02:51.536385Z" |
| } |
| }, |
| "cell_type": "code", |
| "source": [ |
| "start = time.time()\n", |
| "r = dr.execute([\"s\", \"x\", \"a\"])\n", |
| "print(\"got return from dr\")\n", |
| "print(r)\n", |
| "print(\"Time taken with\", time.time() - start)" |
| ], |
| "id": "960f9f5d5f018b38", |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "z\n", |
| "y\n", |
| "x\n", |
| "a\n", |
| "b\n", |
| "c\n", |
| "d\n", |
| "e\n", |
| "s\n", |
| "got return from dr\n", |
| "{'s': 'z y a b d', 'x': 'z y', 'a': 'a'}\n", |
| "Time taken with 27.080925941467285\n" |
| ] |
| } |
| ], |
| "execution_count": 7 |
| }, |
| { |
| "metadata": {}, |
| "cell_type": "markdown", |
| "source": "# Run it with the adapter -- note the parallelism & time taken", |
| "id": "8a1d2b183b914034" |
| }, |
| { |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:03:18.904861Z", |
| "start_time": "2025-01-02T05:03:18.632385Z" |
| } |
| }, |
| "cell_type": "code", |
| "source": [ |
| "from hamilton import driver\n", |
| "from hamilton.plugins import h_threadpool\n", |
| "\n", |
| "adapter = h_threadpool.FutureAdapter()\n", |
| "dr = driver.Builder().with_modules(my_functions).with_adapters(adapter).build()\n", |
| "dr" |
| ], |
| "id": "63853f111ef28439", |
| "outputs": [ |
| { |
| "data": { |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n -->\n<!-- Pages: 1 -->\n<svg width=\"342pt\" height=\"359pt\"\n viewBox=\"0.00 0.00 341.85 358.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 354.6)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-354.6 337.85,-354.6 337.85,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster__legend</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n</g>\n<!-- c -->\n<g id=\"node1\" class=\"node\">\n<title>c</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-104.6C155.85,-104.6 125.85,-104.6 125.85,-104.6 119.85,-104.6 113.85,-98.6 113.85,-92.6 113.85,-92.6 113.85,-53 113.85,-53 113.85,-47 119.85,-41 125.85,-41 125.85,-41 155.85,-41 155.85,-41 161.85,-41 167.85,-47 167.85,-53 167.85,-53 167.85,-92.6 167.85,-92.6 167.85,-98.6 161.85,-104.6 155.85,-104.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">c</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- e -->\n<g id=\"node4\" class=\"node\">\n<title>e</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-186.6C238.85,-186.6 208.85,-186.6 208.85,-186.6 202.85,-186.6 196.85,-180.6 196.85,-174.6 196.85,-174.6 196.85,-135 196.85,-135 196.85,-129 202.85,-123 208.85,-123 208.85,-123 238.85,-123 238.85,-123 244.85,-123 250.85,-129 250.85,-135 250.85,-135 250.85,-174.6 250.85,-174.6 250.85,-180.6 244.85,-186.6 238.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">e</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- c->e -->\n<g id=\"edge5\" class=\"edge\">\n<title>c->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-99.2C174.51,-105.86 181.67,-113.11 188.59,-120.12\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.97,-122.44 195.48,-127.09 190.95,-117.52 185.97,-122.44\"/>\n</g>\n<!-- x -->\n<g id=\"node2\" class=\"node\">\n<title>x</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-268.6C238.85,-268.6 208.85,-268.6 208.85,-268.6 202.85,-268.6 196.85,-262.6 196.85,-256.6 196.85,-256.6 196.85,-217 196.85,-217 196.85,-211 202.85,-205 208.85,-205 208.85,-205 238.85,-205 238.85,-205 244.85,-205 250.85,-211 250.85,-217 250.85,-217 250.85,-256.6 250.85,-256.6 250.85,-262.6 244.85,-268.6 238.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">x</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- s -->\n<g id=\"node6\" class=\"node\">\n<title>s</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M321.85,-227.6C321.85,-227.6 291.85,-227.6 291.85,-227.6 285.85,-227.6 279.85,-221.6 279.85,-215.6 279.85,-215.6 279.85,-176 279.85,-176 279.85,-170 285.85,-164 291.85,-164 291.85,-164 321.85,-164 321.85,-164 327.85,-164 333.85,-170 333.85,-176 333.85,-176 333.85,-215.6 333.85,-215.6 333.85,-221.6 327.85,-227.6 321.85,-227.6\"/>\n<text text-anchor=\"start\" x=\"303.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">s</text>\n<text text-anchor=\"start\" x=\"299.35\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- x->s -->\n<g id=\"edge7\" class=\"edge\">\n<title>x->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-223.6C256.79,-220.63 263.12,-217.43 269.33,-214.29\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"270.85,-217.44 278.2,-209.8 267.69,-211.19 270.85,-217.44\"/>\n</g>\n<!-- a -->\n<g id=\"node3\" class=\"node\">\n<title>a</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-145.6C65.43,-145.6 35.43,-145.6 35.43,-145.6 29.43,-145.6 23.43,-139.6 23.43,-133.6 23.43,-133.6 23.43,-94 23.43,-94 23.43,-88 29.43,-82 35.43,-82 35.43,-82 65.43,-82 65.43,-82 71.43,-82 77.43,-88 77.43,-94 77.43,-94 77.43,-133.6 77.43,-133.6 77.43,-139.6 71.43,-145.6 65.43,-145.6\"/>\n<text text-anchor=\"start\" x=\"46.68\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- a->c -->\n<g id=\"edge1\" class=\"edge\">\n<title>a->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-101.72C85.52,-97.99 94.53,-93.81 103.15,-89.82\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"104.4,-93.1 112,-85.71 101.45,-86.75 104.4,-93.1\"/>\n</g>\n<!-- e->s -->\n<g id=\"edge8\" class=\"edge\">\n<title>e->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-168C256.79,-170.97 263.12,-174.17 269.33,-177.31\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.69,-180.41 278.2,-181.8 270.85,-174.16 267.69,-180.41\"/>\n</g>\n<!-- b -->\n<g id=\"node5\" class=\"node\">\n<title>b</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-63.6C65.43,-63.6 35.43,-63.6 35.43,-63.6 29.43,-63.6 23.43,-57.6 23.43,-51.6 23.43,-51.6 23.43,-12 23.43,-12 23.43,-6 29.43,0 35.43,0 35.43,0 65.43,0 65.43,0 71.43,0 77.43,-6 77.43,-12 77.43,-12 77.43,-51.6 77.43,-51.6 77.43,-57.6 71.43,-63.6 65.43,-63.6\"/>\n<text text-anchor=\"start\" x=\"46.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">b</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- b->c -->\n<g id=\"edge2\" class=\"edge\">\n<title>b->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-43.88C85.52,-47.61 94.53,-51.79 103.15,-55.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"101.45,-58.85 112,-59.89 104.4,-52.5 101.45,-58.85\"/>\n</g>\n<!-- z -->\n<g id=\"node7\" class=\"node\">\n<title>z</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-350.6C155.85,-350.6 125.85,-350.6 125.85,-350.6 119.85,-350.6 113.85,-344.6 113.85,-338.6 113.85,-338.6 113.85,-299 113.85,-299 113.85,-293 119.85,-287 125.85,-287 125.85,-287 155.85,-287 155.85,-287 161.85,-287 167.85,-293 167.85,-299 167.85,-299 167.85,-338.6 167.85,-338.6 167.85,-344.6 161.85,-350.6 155.85,-350.6\"/>\n<text text-anchor=\"start\" x=\"137.48\" y=\"-327.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">z</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-299.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- z->x -->\n<g id=\"edge3\" class=\"edge\">\n<title>z->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-292.4C174.51,-285.74 181.67,-278.49 188.59,-271.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"190.95,-274.08 195.48,-264.51 185.97,-269.16 190.95,-274.08\"/>\n</g>\n<!-- d -->\n<g id=\"node8\" class=\"node\">\n<title>d</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-186.6C155.85,-186.6 125.85,-186.6 125.85,-186.6 119.85,-186.6 113.85,-180.6 113.85,-174.6 113.85,-174.6 113.85,-135 113.85,-135 113.85,-129 119.85,-123 125.85,-123 125.85,-123 155.85,-123 155.85,-123 161.85,-123 167.85,-129 167.85,-135 167.85,-135 167.85,-174.6 167.85,-174.6 167.85,-180.6 161.85,-186.6 155.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"136.73\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">d</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- d->e -->\n<g id=\"edge6\" class=\"edge\">\n<title>d->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-154.8C173.5,-154.8 179.51,-154.8 185.42,-154.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-158.3 195.03,-154.8 185.03,-151.3 185.03,-158.3\"/>\n</g>\n<!-- y -->\n<g id=\"node9\" class=\"node\">\n<title>y</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-268.6C155.85,-268.6 125.85,-268.6 125.85,-268.6 119.85,-268.6 113.85,-262.6 113.85,-256.6 113.85,-256.6 113.85,-217 113.85,-217 113.85,-211 119.85,-205 125.85,-205 125.85,-205 155.85,-205 155.85,-205 161.85,-205 167.85,-211 167.85,-217 167.85,-217 167.85,-256.6 167.85,-256.6 167.85,-262.6 161.85,-268.6 155.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">y</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- y->x -->\n<g id=\"edge4\" class=\"edge\">\n<title>y->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-236.8C173.5,-236.8 179.51,-236.8 185.42,-236.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-240.3 195.03,-236.8 185.03,-233.3 185.03,-240.3\"/>\n</g>\n<!-- function -->\n<g id=\"node10\" class=\"node\">\n<title>function</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n</g>\n</g>\n</svg>\n", |
| "text/plain": [ |
| "<hamilton.driver.Driver at 0x1508d2df0>" |
| ] |
| }, |
| "execution_count": 8, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "execution_count": 8 |
| }, |
| { |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:03:30.949086Z", |
| "start_time": "2025-01-02T05:03:18.925667Z" |
| } |
| }, |
| "cell_type": "code", |
| "source": [ |
| "start = time.time()\n", |
| "r = dr.execute([\"s\", \"x\", \"a\"])\n", |
| "print(\"got return from dr\")\n", |
| "print(r)\n", |
| "print(\"Time taken with\", time.time() - start)" |
| ], |
| "id": "1bb057f4277705de", |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "z\n", |
| "y\n", |
| "a\n", |
| "b\n", |
| "d\n", |
| "x\n", |
| "c\n", |
| "e\n", |
| "s\n", |
| "got return from dr\n", |
| "{'s': 'z y a b d', 'x': 'z y', 'a': 'a'}\n", |
| "Time taken with 12.019250869750977\n" |
| ] |
| } |
| ], |
| "execution_count": 9 |
| }, |
| { |
| "metadata": {}, |
| "cell_type": "markdown", |
| "source": [ |
| "# 27 seconds vs 12 seconds\n", |
| "\n", |
| "With the adapter we see a significant improvement in time taken to execute the DAG. This is because the adapter is able to parallelize the execution." |
| ], |
| "id": "56e9fb7445639984" |
| }, |
| { |
| "metadata": {}, |
| "cell_type": "code", |
| "outputs": [], |
| "execution_count": null, |
| "source": "", |
| "id": "e31132a4fd211887" |
| } |
| ], |
| "metadata": { |
| "kernelspec": { |
| "display_name": "Python 3", |
| "language": "python", |
| "name": "python3" |
| }, |
| "language_info": { |
| "codemirror_mode": { |
| "name": "ipython", |
| "version": 2 |
| }, |
| "file_extension": ".py", |
| "mimetype": "text/x-python", |
| "name": "python", |
| "nbconvert_exporter": "python", |
| "pygments_lexer": "ipython2", |
| "version": "2.7.6" |
| } |
| }, |
| "nbformat": 4, |
| "nbformat_minor": 5 |
| } |
