| { |
| "cells": [ |
| { |
| "cell_type": "markdown", |
| "id": "7fb27b941602401d91542211134fc71a", |
| "metadata": {}, |
| "source": [ |
| "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "id": "initial_id", |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "# Execute this cell to install dependencies\n", |
| "%pip install apache-hamilton[visualization]" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "id": "7b55978b426b6e42", |
| "metadata": {}, |
| "source": "# run me in google colab [](https://colab.research.google.com/github/dagworks-inc/hamilton/blob/main/examples/parallelism/lazy_threadpool_execution/notebook.ipynb) [](https://github.com/apache/hamilton/blob/main/examples/parallelism/lazy_threadpool_execution/notebook.ipynb)\n" |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 1, |
| "id": "a1f2f8937a0b0488", |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:00:01.808537Z", |
| "start_time": "2025-01-02T04:59:53.847872Z" |
| } |
| }, |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "/Users/stefankrawczyk/.pyenv/versions/knowledge_retrieval-py39/lib/python3.9/site-packages/pyspark/pandas/__init__.py:50: UserWarning: 'PYARROW_IGNORE_TIMEZONE' environment variable was not set. It is required to set this environment variable to '1' in both driver and executor sides if you use pyarrow>=2.0.0. pandas-on-Spark will set it for you but it does not work if there is a Spark context already launched.\n", |
| " warnings.warn(\n" |
| ] |
| } |
| ], |
| "source": [ |
| "%load_ext hamilton.plugins.jupyter_magic" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "id": "32c01561386fc348", |
| "metadata": {}, |
| "source": [ |
| "# Create a module with some functions\n", |
| "This hopefully shows a good example of what could be parallelized given the structure of the DAG." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 2, |
| "id": "8e0e3b7a96ca1d44", |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:00:16.514305Z", |
| "start_time": "2025-01-02T05:00:16.101379Z" |
| } |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n -->\n<!-- Pages: 1 -->\n<svg width=\"342pt\" height=\"359pt\"\n viewBox=\"0.00 0.00 341.85 358.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 354.6)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-354.6 337.85,-354.6 337.85,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster__legend</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n</g>\n<!-- c -->\n<g id=\"node1\" class=\"node\">\n<title>c</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-104.6C155.85,-104.6 125.85,-104.6 125.85,-104.6 119.85,-104.6 113.85,-98.6 113.85,-92.6 113.85,-92.6 113.85,-53 113.85,-53 113.85,-47 119.85,-41 125.85,-41 125.85,-41 155.85,-41 155.85,-41 161.85,-41 167.85,-47 167.85,-53 167.85,-53 167.85,-92.6 167.85,-92.6 167.85,-98.6 161.85,-104.6 155.85,-104.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">c</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- e -->\n<g id=\"node4\" class=\"node\">\n<title>e</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-186.6C238.85,-186.6 208.85,-186.6 208.85,-186.6 202.85,-186.6 196.85,-180.6 196.85,-174.6 196.85,-174.6 196.85,-135 196.85,-135 196.85,-129 202.85,-123 208.85,-123 208.85,-123 238.85,-123 238.85,-123 244.85,-123 250.85,-129 250.85,-135 250.85,-135 250.85,-174.6 250.85,-174.6 250.85,-180.6 244.85,-186.6 238.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">e</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- c->e -->\n<g id=\"edge5\" class=\"edge\">\n<title>c->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-99.2C174.51,-105.86 181.67,-113.11 188.59,-120.12\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.97,-122.44 195.48,-127.09 190.95,-117.52 185.97,-122.44\"/>\n</g>\n<!-- x -->\n<g id=\"node2\" class=\"node\">\n<title>x</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-268.6C238.85,-268.6 208.85,-268.6 208.85,-268.6 202.85,-268.6 196.85,-262.6 196.85,-256.6 196.85,-256.6 196.85,-217 196.85,-217 196.85,-211 202.85,-205 208.85,-205 208.85,-205 238.85,-205 238.85,-205 244.85,-205 250.85,-211 250.85,-217 250.85,-217 250.85,-256.6 250.85,-256.6 250.85,-262.6 244.85,-268.6 238.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">x</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- s -->\n<g id=\"node6\" class=\"node\">\n<title>s</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M321.85,-227.6C321.85,-227.6 291.85,-227.6 291.85,-227.6 285.85,-227.6 279.85,-221.6 279.85,-215.6 279.85,-215.6 279.85,-176 279.85,-176 279.85,-170 285.85,-164 291.85,-164 291.85,-164 321.85,-164 321.85,-164 327.85,-164 333.85,-170 333.85,-176 333.85,-176 333.85,-215.6 333.85,-215.6 333.85,-221.6 327.85,-227.6 321.85,-227.6\"/>\n<text text-anchor=\"start\" x=\"303.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">s</text>\n<text text-anchor=\"start\" x=\"299.35\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- x->s -->\n<g id=\"edge7\" class=\"edge\">\n<title>x->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-223.6C256.79,-220.63 263.12,-217.43 269.33,-214.29\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"270.85,-217.44 278.2,-209.8 267.69,-211.19 270.85,-217.44\"/>\n</g>\n<!-- a -->\n<g id=\"node3\" class=\"node\">\n<title>a</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-145.6C65.43,-145.6 35.43,-145.6 35.43,-145.6 29.43,-145.6 23.43,-139.6 23.43,-133.6 23.43,-133.6 23.43,-94 23.43,-94 23.43,-88 29.43,-82 35.43,-82 35.43,-82 65.43,-82 65.43,-82 71.43,-82 77.43,-88 77.43,-94 77.43,-94 77.43,-133.6 77.43,-133.6 77.43,-139.6 71.43,-145.6 65.43,-145.6\"/>\n<text text-anchor=\"start\" x=\"46.68\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- a->c -->\n<g id=\"edge1\" class=\"edge\">\n<title>a->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-101.72C85.52,-97.99 94.53,-93.81 103.15,-89.82\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"104.4,-93.1 112,-85.71 101.45,-86.75 104.4,-93.1\"/>\n</g>\n<!-- e->s -->\n<g id=\"edge8\" class=\"edge\">\n<title>e->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-168C256.79,-170.97 263.12,-174.17 269.33,-177.31\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.69,-180.41 278.2,-181.8 270.85,-174.16 267.69,-180.41\"/>\n</g>\n<!-- b -->\n<g id=\"node5\" class=\"node\">\n<title>b</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-63.6C65.43,-63.6 35.43,-63.6 35.43,-63.6 29.43,-63.6 23.43,-57.6 23.43,-51.6 23.43,-51.6 23.43,-12 23.43,-12 23.43,-6 29.43,0 35.43,0 35.43,0 65.43,0 65.43,0 71.43,0 77.43,-6 77.43,-12 77.43,-12 77.43,-51.6 77.43,-51.6 77.43,-57.6 71.43,-63.6 65.43,-63.6\"/>\n<text text-anchor=\"start\" x=\"46.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">b</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- b->c -->\n<g id=\"edge2\" class=\"edge\">\n<title>b->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-43.88C85.52,-47.61 94.53,-51.79 103.15,-55.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"101.45,-58.85 112,-59.89 104.4,-52.5 101.45,-58.85\"/>\n</g>\n<!-- z -->\n<g id=\"node7\" class=\"node\">\n<title>z</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-350.6C155.85,-350.6 125.85,-350.6 125.85,-350.6 119.85,-350.6 113.85,-344.6 113.85,-338.6 113.85,-338.6 113.85,-299 113.85,-299 113.85,-293 119.85,-287 125.85,-287 125.85,-287 155.85,-287 155.85,-287 161.85,-287 167.85,-293 167.85,-299 167.85,-299 167.85,-338.6 167.85,-338.6 167.85,-344.6 161.85,-350.6 155.85,-350.6\"/>\n<text text-anchor=\"start\" x=\"137.48\" y=\"-327.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">z</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-299.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- z->x -->\n<g id=\"edge3\" class=\"edge\">\n<title>z->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-292.4C174.51,-285.74 181.67,-278.49 188.59,-271.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"190.95,-274.08 195.48,-264.51 185.97,-269.16 190.95,-274.08\"/>\n</g>\n<!-- d -->\n<g id=\"node8\" class=\"node\">\n<title>d</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-186.6C155.85,-186.6 125.85,-186.6 125.85,-186.6 119.85,-186.6 113.85,-180.6 113.85,-174.6 113.85,-174.6 113.85,-135 113.85,-135 113.85,-129 119.85,-123 125.85,-123 125.85,-123 155.85,-123 155.85,-123 161.85,-123 167.85,-129 167.85,-135 167.85,-135 167.85,-174.6 167.85,-174.6 167.85,-180.6 161.85,-186.6 155.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"136.73\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">d</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- d->e -->\n<g id=\"edge6\" class=\"edge\">\n<title>d->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-154.8C173.5,-154.8 179.51,-154.8 185.42,-154.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-158.3 195.03,-154.8 185.03,-151.3 185.03,-158.3\"/>\n</g>\n<!-- y -->\n<g id=\"node9\" class=\"node\">\n<title>y</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-268.6C155.85,-268.6 125.85,-268.6 125.85,-268.6 119.85,-268.6 113.85,-262.6 113.85,-256.6 113.85,-256.6 113.85,-217 113.85,-217 113.85,-211 119.85,-205 125.85,-205 125.85,-205 155.85,-205 155.85,-205 161.85,-205 167.85,-211 167.85,-217 167.85,-217 167.85,-256.6 167.85,-256.6 167.85,-262.6 161.85,-268.6 155.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">y</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- y->x -->\n<g id=\"edge4\" class=\"edge\">\n<title>y->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-236.8C173.5,-236.8 179.51,-236.8 185.42,-236.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-240.3 195.03,-236.8 185.03,-233.3 185.03,-240.3\"/>\n</g>\n<!-- function -->\n<g id=\"node10\" class=\"node\">\n<title>function</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n</g>\n</g>\n</svg>\n", |
| "text/plain": [ |
| "<graphviz.graphs.Digraph at 0x14a889520>" |
| ] |
| }, |
| "metadata": {}, |
| "output_type": "display_data" |
| } |
| ], |
| "source": [ |
| "%%cell_to_module my_functions --display\n", |
| "\n", |
| "import time\n", |
| "\n", |
| "\n", |
| "def a() -> str:\n", |
| " print(\"a\")\n", |
| " time.sleep(3)\n", |
| " return \"a\"\n", |
| "\n", |
| "\n", |
| "def b() -> str:\n", |
| " print(\"b\")\n", |
| " time.sleep(3)\n", |
| " return \"b\"\n", |
| "\n", |
| "\n", |
| "def c(a: str, b: str) -> str:\n", |
| " print(\"c\")\n", |
| " time.sleep(3)\n", |
| " return a + \" \" + b\n", |
| "\n", |
| "\n", |
| "def d() -> str:\n", |
| " print(\"d\")\n", |
| " time.sleep(3)\n", |
| " return \"d\"\n", |
| "\n", |
| "\n", |
| "def e(c: str, d: str) -> str:\n", |
| " print(\"e\")\n", |
| " time.sleep(3)\n", |
| " return c + \" \" + d\n", |
| "\n", |
| "\n", |
| "def z() -> str:\n", |
| " print(\"z\")\n", |
| " time.sleep(3)\n", |
| " return \"z\"\n", |
| "\n", |
| "\n", |
| "def y() -> str:\n", |
| " print(\"y\")\n", |
| " time.sleep(3)\n", |
| " return \"y\"\n", |
| "\n", |
| "\n", |
| "def x(z: str, y: str) -> str:\n", |
| " print(\"x\")\n", |
| " time.sleep(3)\n", |
| " return z + \" \" + y\n", |
| "\n", |
| "\n", |
| "def s(x: str, e: str) -> str:\n", |
| " print(\"s\")\n", |
| " time.sleep(3)\n", |
| " return x + \" \" + e\n", |
| "\n" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "id": "60355598274f9b79", |
| "metadata": {}, |
| "source": "# Run it without the adapter" |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 6, |
| "id": "fcb0677daf5b4a31", |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:02:44.719265Z", |
| "start_time": "2025-01-02T05:02:44.423066Z" |
| } |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n -->\n<!-- Pages: 1 -->\n<svg width=\"342pt\" height=\"359pt\"\n viewBox=\"0.00 0.00 341.85 358.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 354.6)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-354.6 337.85,-354.6 337.85,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster__legend</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n</g>\n<!-- c -->\n<g id=\"node1\" class=\"node\">\n<title>c</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-104.6C155.85,-104.6 125.85,-104.6 125.85,-104.6 119.85,-104.6 113.85,-98.6 113.85,-92.6 113.85,-92.6 113.85,-53 113.85,-53 113.85,-47 119.85,-41 125.85,-41 125.85,-41 155.85,-41 155.85,-41 161.85,-41 167.85,-47 167.85,-53 167.85,-53 167.85,-92.6 167.85,-92.6 167.85,-98.6 161.85,-104.6 155.85,-104.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">c</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- e -->\n<g id=\"node4\" class=\"node\">\n<title>e</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-186.6C238.85,-186.6 208.85,-186.6 208.85,-186.6 202.85,-186.6 196.85,-180.6 196.85,-174.6 196.85,-174.6 196.85,-135 196.85,-135 196.85,-129 202.85,-123 208.85,-123 208.85,-123 238.85,-123 238.85,-123 244.85,-123 250.85,-129 250.85,-135 250.85,-135 250.85,-174.6 250.85,-174.6 250.85,-180.6 244.85,-186.6 238.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">e</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- c->e -->\n<g id=\"edge5\" class=\"edge\">\n<title>c->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-99.2C174.51,-105.86 181.67,-113.11 188.59,-120.12\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.97,-122.44 195.48,-127.09 190.95,-117.52 185.97,-122.44\"/>\n</g>\n<!-- x -->\n<g id=\"node2\" class=\"node\">\n<title>x</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-268.6C238.85,-268.6 208.85,-268.6 208.85,-268.6 202.85,-268.6 196.85,-262.6 196.85,-256.6 196.85,-256.6 196.85,-217 196.85,-217 196.85,-211 202.85,-205 208.85,-205 208.85,-205 238.85,-205 238.85,-205 244.85,-205 250.85,-211 250.85,-217 250.85,-217 250.85,-256.6 250.85,-256.6 250.85,-262.6 244.85,-268.6 238.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">x</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- s -->\n<g id=\"node6\" class=\"node\">\n<title>s</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M321.85,-227.6C321.85,-227.6 291.85,-227.6 291.85,-227.6 285.85,-227.6 279.85,-221.6 279.85,-215.6 279.85,-215.6 279.85,-176 279.85,-176 279.85,-170 285.85,-164 291.85,-164 291.85,-164 321.85,-164 321.85,-164 327.85,-164 333.85,-170 333.85,-176 333.85,-176 333.85,-215.6 333.85,-215.6 333.85,-221.6 327.85,-227.6 321.85,-227.6\"/>\n<text text-anchor=\"start\" x=\"303.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">s</text>\n<text text-anchor=\"start\" x=\"299.35\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- x->s -->\n<g id=\"edge7\" class=\"edge\">\n<title>x->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-223.6C256.79,-220.63 263.12,-217.43 269.33,-214.29\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"270.85,-217.44 278.2,-209.8 267.69,-211.19 270.85,-217.44\"/>\n</g>\n<!-- a -->\n<g id=\"node3\" class=\"node\">\n<title>a</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-145.6C65.43,-145.6 35.43,-145.6 35.43,-145.6 29.43,-145.6 23.43,-139.6 23.43,-133.6 23.43,-133.6 23.43,-94 23.43,-94 23.43,-88 29.43,-82 35.43,-82 35.43,-82 65.43,-82 65.43,-82 71.43,-82 77.43,-88 77.43,-94 77.43,-94 77.43,-133.6 77.43,-133.6 77.43,-139.6 71.43,-145.6 65.43,-145.6\"/>\n<text text-anchor=\"start\" x=\"46.68\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- a->c -->\n<g id=\"edge1\" class=\"edge\">\n<title>a->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-101.72C85.52,-97.99 94.53,-93.81 103.15,-89.82\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"104.4,-93.1 112,-85.71 101.45,-86.75 104.4,-93.1\"/>\n</g>\n<!-- e->s -->\n<g id=\"edge8\" class=\"edge\">\n<title>e->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-168C256.79,-170.97 263.12,-174.17 269.33,-177.31\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.69,-180.41 278.2,-181.8 270.85,-174.16 267.69,-180.41\"/>\n</g>\n<!-- b -->\n<g id=\"node5\" class=\"node\">\n<title>b</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-63.6C65.43,-63.6 35.43,-63.6 35.43,-63.6 29.43,-63.6 23.43,-57.6 23.43,-51.6 23.43,-51.6 23.43,-12 23.43,-12 23.43,-6 29.43,0 35.43,0 35.43,0 65.43,0 65.43,0 71.43,0 77.43,-6 77.43,-12 77.43,-12 77.43,-51.6 77.43,-51.6 77.43,-57.6 71.43,-63.6 65.43,-63.6\"/>\n<text text-anchor=\"start\" x=\"46.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">b</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- b->c -->\n<g id=\"edge2\" class=\"edge\">\n<title>b->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-43.88C85.52,-47.61 94.53,-51.79 103.15,-55.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"101.45,-58.85 112,-59.89 104.4,-52.5 101.45,-58.85\"/>\n</g>\n<!-- z -->\n<g id=\"node7\" class=\"node\">\n<title>z</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-350.6C155.85,-350.6 125.85,-350.6 125.85,-350.6 119.85,-350.6 113.85,-344.6 113.85,-338.6 113.85,-338.6 113.85,-299 113.85,-299 113.85,-293 119.85,-287 125.85,-287 125.85,-287 155.85,-287 155.85,-287 161.85,-287 167.85,-293 167.85,-299 167.85,-299 167.85,-338.6 167.85,-338.6 167.85,-344.6 161.85,-350.6 155.85,-350.6\"/>\n<text text-anchor=\"start\" x=\"137.48\" y=\"-327.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">z</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-299.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- z->x -->\n<g id=\"edge3\" class=\"edge\">\n<title>z->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-292.4C174.51,-285.74 181.67,-278.49 188.59,-271.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"190.95,-274.08 195.48,-264.51 185.97,-269.16 190.95,-274.08\"/>\n</g>\n<!-- d -->\n<g id=\"node8\" class=\"node\">\n<title>d</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-186.6C155.85,-186.6 125.85,-186.6 125.85,-186.6 119.85,-186.6 113.85,-180.6 113.85,-174.6 113.85,-174.6 113.85,-135 113.85,-135 113.85,-129 119.85,-123 125.85,-123 125.85,-123 155.85,-123 155.85,-123 161.85,-123 167.85,-129 167.85,-135 167.85,-135 167.85,-174.6 167.85,-174.6 167.85,-180.6 161.85,-186.6 155.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"136.73\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">d</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- d->e -->\n<g id=\"edge6\" class=\"edge\">\n<title>d->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-154.8C173.5,-154.8 179.51,-154.8 185.42,-154.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-158.3 195.03,-154.8 185.03,-151.3 185.03,-158.3\"/>\n</g>\n<!-- y -->\n<g id=\"node9\" class=\"node\">\n<title>y</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-268.6C155.85,-268.6 125.85,-268.6 125.85,-268.6 119.85,-268.6 113.85,-262.6 113.85,-256.6 113.85,-256.6 113.85,-217 113.85,-217 113.85,-211 119.85,-205 125.85,-205 125.85,-205 155.85,-205 155.85,-205 161.85,-205 167.85,-211 167.85,-217 167.85,-217 167.85,-256.6 167.85,-256.6 167.85,-262.6 161.85,-268.6 155.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">y</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- y->x -->\n<g id=\"edge4\" class=\"edge\">\n<title>y->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-236.8C173.5,-236.8 179.51,-236.8 185.42,-236.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-240.3 195.03,-236.8 185.03,-233.3 185.03,-240.3\"/>\n</g>\n<!-- function -->\n<g id=\"node10\" class=\"node\">\n<title>function</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n</g>\n</g>\n</svg>\n", |
| "text/plain": [ |
| "<hamilton.driver.Driver at 0x1508cca90>" |
| ] |
| }, |
| "execution_count": 6, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "from hamilton import driver\n", |
| "\n", |
| "dr = driver.Builder().with_modules(my_functions).build()\n", |
| "dr" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 7, |
| "id": "960f9f5d5f018b38", |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:03:18.620774Z", |
| "start_time": "2025-01-02T05:02:51.536385Z" |
| } |
| }, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "z\n", |
| "y\n", |
| "x\n", |
| "a\n", |
| "b\n", |
| "c\n", |
| "d\n", |
| "e\n", |
| "s\n", |
| "got return from dr\n", |
| "{'s': 'z y a b d', 'x': 'z y', 'a': 'a'}\n", |
| "Time taken with 27.080925941467285\n" |
| ] |
| } |
| ], |
| "source": [ |
| "start = time.time()\n", |
| "r = dr.execute([\"s\", \"x\", \"a\"])\n", |
| "print(\"got return from dr\")\n", |
| "print(r)\n", |
| "print(\"Time taken with\", time.time() - start)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "id": "8a1d2b183b914034", |
| "metadata": {}, |
| "source": "# Run it with the adapter -- note the parallelism & time taken" |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 8, |
| "id": "63853f111ef28439", |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:03:18.904861Z", |
| "start_time": "2025-01-02T05:03:18.632385Z" |
| } |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 12.0.0 (20240704.0754)\n -->\n<!-- Pages: 1 -->\n<svg width=\"342pt\" height=\"359pt\"\n viewBox=\"0.00 0.00 341.85 358.60\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 354.6)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-354.6 337.85,-354.6 337.85,4 -4,4\"/>\n<g id=\"clust1\" class=\"cluster\">\n<title>cluster__legend</title>\n<polygon fill=\"#ffffff\" stroke=\"black\" points=\"8,-155.8 8,-230.8 92.85,-230.8 92.85,-155.8 8,-155.8\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-213.5\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">Legend</text>\n</g>\n<!-- c -->\n<g id=\"node1\" class=\"node\">\n<title>c</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-104.6C155.85,-104.6 125.85,-104.6 125.85,-104.6 119.85,-104.6 113.85,-98.6 113.85,-92.6 113.85,-92.6 113.85,-53 113.85,-53 113.85,-47 119.85,-41 125.85,-41 125.85,-41 155.85,-41 155.85,-41 161.85,-41 167.85,-47 167.85,-53 167.85,-53 167.85,-92.6 167.85,-92.6 167.85,-98.6 161.85,-104.6 155.85,-104.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-81.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">c</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-53.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- e -->\n<g id=\"node4\" class=\"node\">\n<title>e</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-186.6C238.85,-186.6 208.85,-186.6 208.85,-186.6 202.85,-186.6 196.85,-180.6 196.85,-174.6 196.85,-174.6 196.85,-135 196.85,-135 196.85,-129 202.85,-123 208.85,-123 208.85,-123 238.85,-123 238.85,-123 244.85,-123 250.85,-129 250.85,-135 250.85,-135 250.85,-174.6 250.85,-174.6 250.85,-180.6 244.85,-186.6 238.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">e</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- c->e -->\n<g id=\"edge5\" class=\"edge\">\n<title>c->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-99.2C174.51,-105.86 181.67,-113.11 188.59,-120.12\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.97,-122.44 195.48,-127.09 190.95,-117.52 185.97,-122.44\"/>\n</g>\n<!-- x -->\n<g id=\"node2\" class=\"node\">\n<title>x</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M238.85,-268.6C238.85,-268.6 208.85,-268.6 208.85,-268.6 202.85,-268.6 196.85,-262.6 196.85,-256.6 196.85,-256.6 196.85,-217 196.85,-217 196.85,-211 202.85,-205 208.85,-205 208.85,-205 238.85,-205 238.85,-205 244.85,-205 250.85,-211 250.85,-217 250.85,-217 250.85,-256.6 250.85,-256.6 250.85,-262.6 244.85,-268.6 238.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"220.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">x</text>\n<text text-anchor=\"start\" x=\"216.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- s -->\n<g id=\"node6\" class=\"node\">\n<title>s</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M321.85,-227.6C321.85,-227.6 291.85,-227.6 291.85,-227.6 285.85,-227.6 279.85,-221.6 279.85,-215.6 279.85,-215.6 279.85,-176 279.85,-176 279.85,-170 285.85,-164 291.85,-164 291.85,-164 321.85,-164 321.85,-164 327.85,-164 333.85,-170 333.85,-176 333.85,-176 333.85,-215.6 333.85,-215.6 333.85,-221.6 327.85,-227.6 321.85,-227.6\"/>\n<text text-anchor=\"start\" x=\"303.1\" y=\"-204.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">s</text>\n<text text-anchor=\"start\" x=\"299.35\" y=\"-176.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- x->s -->\n<g id=\"edge7\" class=\"edge\">\n<title>x->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-223.6C256.79,-220.63 263.12,-217.43 269.33,-214.29\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"270.85,-217.44 278.2,-209.8 267.69,-211.19 270.85,-217.44\"/>\n</g>\n<!-- a -->\n<g id=\"node3\" class=\"node\">\n<title>a</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-145.6C65.43,-145.6 35.43,-145.6 35.43,-145.6 29.43,-145.6 23.43,-139.6 23.43,-133.6 23.43,-133.6 23.43,-94 23.43,-94 23.43,-88 29.43,-82 35.43,-82 35.43,-82 65.43,-82 65.43,-82 71.43,-82 77.43,-88 77.43,-94 77.43,-94 77.43,-133.6 77.43,-133.6 77.43,-139.6 71.43,-145.6 65.43,-145.6\"/>\n<text text-anchor=\"start\" x=\"46.68\" y=\"-122.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">a</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-94.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- a->c -->\n<g id=\"edge1\" class=\"edge\">\n<title>a->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-101.72C85.52,-97.99 94.53,-93.81 103.15,-89.82\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"104.4,-93.1 112,-85.71 101.45,-86.75 104.4,-93.1\"/>\n</g>\n<!-- e->s -->\n<g id=\"edge8\" class=\"edge\">\n<title>e->s</title>\n<path fill=\"none\" stroke=\"black\" d=\"M250.92,-168C256.79,-170.97 263.12,-174.17 269.33,-177.31\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"267.69,-180.41 278.2,-181.8 270.85,-174.16 267.69,-180.41\"/>\n</g>\n<!-- b -->\n<g id=\"node5\" class=\"node\">\n<title>b</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M65.43,-63.6C65.43,-63.6 35.43,-63.6 35.43,-63.6 29.43,-63.6 23.43,-57.6 23.43,-51.6 23.43,-51.6 23.43,-12 23.43,-12 23.43,-6 29.43,0 35.43,0 35.43,0 65.43,0 65.43,0 71.43,0 77.43,-6 77.43,-12 77.43,-12 77.43,-51.6 77.43,-51.6 77.43,-57.6 71.43,-63.6 65.43,-63.6\"/>\n<text text-anchor=\"start\" x=\"46.3\" y=\"-40.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">b</text>\n<text text-anchor=\"start\" x=\"42.93\" y=\"-12.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- b->c -->\n<g id=\"edge2\" class=\"edge\">\n<title>b->c</title>\n<path fill=\"none\" stroke=\"black\" d=\"M77.48,-43.88C85.52,-47.61 94.53,-51.79 103.15,-55.78\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"101.45,-58.85 112,-59.89 104.4,-52.5 101.45,-58.85\"/>\n</g>\n<!-- z -->\n<g id=\"node7\" class=\"node\">\n<title>z</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-350.6C155.85,-350.6 125.85,-350.6 125.85,-350.6 119.85,-350.6 113.85,-344.6 113.85,-338.6 113.85,-338.6 113.85,-299 113.85,-299 113.85,-293 119.85,-287 125.85,-287 125.85,-287 155.85,-287 155.85,-287 161.85,-287 167.85,-293 167.85,-299 167.85,-299 167.85,-338.6 167.85,-338.6 167.85,-344.6 161.85,-350.6 155.85,-350.6\"/>\n<text text-anchor=\"start\" x=\"137.48\" y=\"-327.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">z</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-299.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- z->x -->\n<g id=\"edge3\" class=\"edge\">\n<title>z->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-292.4C174.51,-285.74 181.67,-278.49 188.59,-271.48\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"190.95,-274.08 195.48,-264.51 185.97,-269.16 190.95,-274.08\"/>\n</g>\n<!-- d -->\n<g id=\"node8\" class=\"node\">\n<title>d</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-186.6C155.85,-186.6 125.85,-186.6 125.85,-186.6 119.85,-186.6 113.85,-180.6 113.85,-174.6 113.85,-174.6 113.85,-135 113.85,-135 113.85,-129 119.85,-123 125.85,-123 125.85,-123 155.85,-123 155.85,-123 161.85,-123 167.85,-129 167.85,-135 167.85,-135 167.85,-174.6 167.85,-174.6 167.85,-180.6 161.85,-186.6 155.85,-186.6\"/>\n<text text-anchor=\"start\" x=\"136.73\" y=\"-163.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">d</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-135.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- d->e -->\n<g id=\"edge6\" class=\"edge\">\n<title>d->e</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-154.8C173.5,-154.8 179.51,-154.8 185.42,-154.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-158.3 195.03,-154.8 185.03,-151.3 185.03,-158.3\"/>\n</g>\n<!-- y -->\n<g id=\"node9\" class=\"node\">\n<title>y</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M155.85,-268.6C155.85,-268.6 125.85,-268.6 125.85,-268.6 119.85,-268.6 113.85,-262.6 113.85,-256.6 113.85,-256.6 113.85,-217 113.85,-217 113.85,-211 119.85,-205 125.85,-205 125.85,-205 155.85,-205 155.85,-205 161.85,-205 167.85,-211 167.85,-217 167.85,-217 167.85,-256.6 167.85,-256.6 167.85,-262.6 161.85,-268.6 155.85,-268.6\"/>\n<text text-anchor=\"start\" x=\"137.1\" y=\"-245.5\" font-family=\"Helvetica,sans-Serif\" font-weight=\"bold\" font-size=\"14.00\">y</text>\n<text text-anchor=\"start\" x=\"133.35\" y=\"-217.5\" font-family=\"Helvetica,sans-Serif\" font-style=\"italic\" font-size=\"14.00\">str</text>\n</g>\n<!-- y->x -->\n<g id=\"edge4\" class=\"edge\">\n<title>y->x</title>\n<path fill=\"none\" stroke=\"black\" d=\"M167.92,-236.8C173.5,-236.8 179.51,-236.8 185.42,-236.8\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"185.03,-240.3 195.03,-236.8 185.03,-233.3 185.03,-240.3\"/>\n</g>\n<!-- function -->\n<g id=\"node10\" class=\"node\">\n<title>function</title>\n<path fill=\"#b4d8e4\" stroke=\"black\" d=\"M72.85,-200.1C72.85,-200.1 28,-200.1 28,-200.1 22,-200.1 16,-194.1 16,-188.1 16,-188.1 16,-175.5 16,-175.5 16,-169.5 22,-163.5 28,-163.5 28,-163.5 72.85,-163.5 72.85,-163.5 78.85,-163.5 84.85,-169.5 84.85,-175.5 84.85,-175.5 84.85,-188.1 84.85,-188.1 84.85,-194.1 78.85,-200.1 72.85,-200.1\"/>\n<text text-anchor=\"middle\" x=\"50.43\" y=\"-176\" font-family=\"Helvetica,sans-Serif\" font-size=\"14.00\">function</text>\n</g>\n</g>\n</svg>\n", |
| "text/plain": [ |
| "<hamilton.driver.Driver at 0x1508d2df0>" |
| ] |
| }, |
| "execution_count": 8, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "from hamilton import driver\n", |
| "from hamilton.plugins import h_threadpool\n", |
| "\n", |
| "adapter = h_threadpool.FutureAdapter()\n", |
| "dr = driver.Builder().with_modules(my_functions).with_adapters(adapter).build()\n", |
| "dr" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 9, |
| "id": "1bb057f4277705de", |
| "metadata": { |
| "ExecuteTime": { |
| "end_time": "2025-01-02T05:03:30.949086Z", |
| "start_time": "2025-01-02T05:03:18.925667Z" |
| } |
| }, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "z\n", |
| "y\n", |
| "a\n", |
| "b\n", |
| "d\n", |
| "x\n", |
| "c\n", |
| "e\n", |
| "s\n", |
| "got return from dr\n", |
| "{'s': 'z y a b d', 'x': 'z y', 'a': 'a'}\n", |
| "Time taken with 12.019250869750977\n" |
| ] |
| } |
| ], |
| "source": [ |
| "start = time.time()\n", |
| "r = dr.execute([\"s\", \"x\", \"a\"])\n", |
| "print(\"got return from dr\")\n", |
| "print(r)\n", |
| "print(\"Time taken with\", time.time() - start)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "id": "56e9fb7445639984", |
| "metadata": {}, |
| "source": [ |
| "# 27 seconds vs 12 seconds\n", |
| "\n", |
| "With the adapter we see a significant improvement in time taken to execute the DAG. This is because the adapter is able to parallelize the execution." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "id": "e31132a4fd211887", |
| "metadata": {}, |
| "outputs": [], |
| "source": [] |
| } |
| ], |
| "metadata": { |
| "kernelspec": { |
| "display_name": "Python 3", |
| "language": "python", |
| "name": "python3" |
| }, |
| "language_info": { |
| "codemirror_mode": { |
| "name": "ipython", |
| "version": 2 |
| }, |
| "file_extension": ".py", |
| "mimetype": "text/x-python", |
| "name": "python", |
| "nbconvert_exporter": "python", |
| "pygments_lexer": "ipython2", |
| "version": "2.7.6" |
| } |
| }, |
| "nbformat": 4, |
| "nbformat_minor": 5 |
| } |
