blob: 9319361b7ac79fea09625168563189f2e6a9708a [file]
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "1ce738e6-14b4-4892-9f2d-a1db94cfb29a",
"metadata": {},
"outputs": [],
"source": [
"!pip install burr[start]"
]
},
{
"cell_type": "markdown",
"id": "2118a2f4-d969-4771-81d3-156b432d1dc8",
"metadata": {},
"source": [
"# Streaming applications\n",
"\n",
"This shows how one goes about working with a streaming response with Burr. \n",
"The code for implementation is in [application.py](application.py)."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "5b53de32-86af-475f-8976-e88a08986e34",
"metadata": {},
"outputs": [],
"source": [
"from application import application as streaming_application\n",
"from application import TERMINAL_ACTIONS"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "f2525edd-9f0c-45c3-84c0-d97512620eb8",
"metadata": {},
"outputs": [],
"source": [
"app = streaming_application()"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "81799877-8f0f-4572-9a96-f6ce6c430d9a",
"metadata": {},
"outputs": [
{
"data": {
"image/svg+xml": [
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
"<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
" \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
"<!-- Generated by graphviz version 8.1.0 (20230707.0739)\n",
" -->\n",
"<!-- Pages: 1 -->\n",
"<svg width=\"529pt\" height=\"304pt\"\n",
" viewBox=\"0.00 0.00 529.38 304.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
"<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 300)\">\n",
"<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-300 525.38,-300 525.38,4 -4,4\"/>\n",
"<!-- prompt -->\n",
"<g id=\"node1\" class=\"node\">\n",
"<title>prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M244.5,-231C244.5,-231 212.75,-231 212.75,-231 206.75,-231 200.75,-225 200.75,-219 200.75,-219 200.75,-207 200.75,-207 200.75,-201 206.75,-195 212.75,-195 212.75,-195 244.5,-195 244.5,-195 250.5,-195 256.5,-201 256.5,-207 256.5,-207 256.5,-219 256.5,-219 256.5,-225 250.5,-231 244.5,-231\"/>\n",
"<text text-anchor=\"middle\" x=\"228.62\" y=\"-207.95\" font-family=\"Times,serif\" font-size=\"14.00\">prompt</text>\n",
"</g>\n",
"<!-- check_safety -->\n",
"<g id=\"node3\" class=\"node\">\n",
"<title>check_safety</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M124.88,-166C124.88,-166 62.38,-166 62.38,-166 56.38,-166 50.38,-160 50.38,-154 50.38,-154 50.38,-142 50.38,-142 50.38,-136 56.38,-130 62.38,-130 62.38,-130 124.88,-130 124.88,-130 130.88,-130 136.88,-136 136.88,-142 136.88,-142 136.88,-154 136.88,-154 136.88,-160 130.88,-166 124.88,-166\"/>\n",
"<text text-anchor=\"middle\" x=\"93.62\" y=\"-142.95\" font-family=\"Times,serif\" font-size=\"14.00\">check_safety</text>\n",
"</g>\n",
"<!-- prompt&#45;&gt;check_safety -->\n",
"<g id=\"edge2\" class=\"edge\">\n",
"<title>prompt&#45;&gt;check_safety</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M200.32,-198.79C183.13,-190.77 160.74,-180.32 140.91,-171.07\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"142.66,-167.55 132.12,-166.5 139.7,-173.9 142.66,-167.55\"/>\n",
"</g>\n",
"<!-- input__prompt -->\n",
"<g id=\"node2\" class=\"node\">\n",
"<title>input__prompt</title>\n",
"<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"228.62\" cy=\"-278\" rx=\"62.1\" ry=\"18\"/>\n",
"<text text-anchor=\"middle\" x=\"228.62\" y=\"-272.95\" font-family=\"Times,serif\" font-size=\"14.00\">input: prompt</text>\n",
"</g>\n",
"<!-- input__prompt&#45;&gt;prompt -->\n",
"<g id=\"edge1\" class=\"edge\">\n",
"<title>input__prompt&#45;&gt;prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M228.62,-259.78C228.62,-254.23 228.62,-247.92 228.62,-241.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"232.13,-242.19 228.62,-232.19 225.13,-242.19 232.13,-242.19\"/>\n",
"</g>\n",
"<!-- unsafe_response -->\n",
"<g id=\"node4\" class=\"node\">\n",
"<title>unsafe_response</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M93.25,-101C93.25,-101 12,-101 12,-101 6,-101 0,-95 0,-89 0,-89 0,-77 0,-77 0,-71 6,-65 12,-65 12,-65 93.25,-65 93.25,-65 99.25,-65 105.25,-71 105.25,-77 105.25,-77 105.25,-89 105.25,-89 105.25,-95 99.25,-101 93.25,-101\"/>\n",
"<text text-anchor=\"middle\" x=\"52.62\" y=\"-77.95\" font-family=\"Times,serif\" font-size=\"14.00\">unsafe_response</text>\n",
"</g>\n",
"<!-- check_safety&#45;&gt;unsafe_response -->\n",
"<g id=\"edge4\" class=\"edge\">\n",
"<title>check_safety&#45;&gt;unsafe_response</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M82.42,-129.78C78.44,-123.67 73.87,-116.65 69.53,-109.97\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"72.2,-108.67 63.81,-102.19 66.34,-112.48 72.2,-108.67\"/>\n",
"</g>\n",
"<!-- decide_mode -->\n",
"<g id=\"node5\" class=\"node\">\n",
"<title>decide_mode</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M253,-101C253,-101 188.25,-101 188.25,-101 182.25,-101 176.25,-95 176.25,-89 176.25,-89 176.25,-77 176.25,-77 176.25,-71 182.25,-65 188.25,-65 188.25,-65 253,-65 253,-65 259,-65 265,-71 265,-77 265,-77 265,-89 265,-89 265,-95 259,-101 253,-101\"/>\n",
"<text text-anchor=\"middle\" x=\"220.62\" y=\"-77.95\" font-family=\"Times,serif\" font-size=\"14.00\">decide_mode</text>\n",
"</g>\n",
"<!-- check_safety&#45;&gt;decide_mode -->\n",
"<g id=\"edge3\" class=\"edge\">\n",
"<title>check_safety&#45;&gt;decide_mode</title>\n",
"<path fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" d=\"M128.66,-129.62C143.38,-122.32 160.66,-113.75 176.25,-106.01\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"177.3,-108.9 184.7,-101.33 174.19,-102.63 177.3,-108.9\"/>\n",
"</g>\n",
"<!-- unsafe_response&#45;&gt;prompt -->\n",
"<g id=\"edge13\" class=\"edge\">\n",
"<title>unsafe_response&#45;&gt;prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M43.61,-101.31C35.59,-119.37 27.18,-147.57 41.62,-166 59.66,-189.01 140.32,-202.05 189.63,-208\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"189.2,-211.59 199.53,-209.26 190,-204.64 189.2,-211.59\"/>\n",
"</g>\n",
"<!-- generate_image -->\n",
"<g id=\"node6\" class=\"node\">\n",
"<title>generate_image</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M141.75,-36C141.75,-36 63.5,-36 63.5,-36 57.5,-36 51.5,-30 51.5,-24 51.5,-24 51.5,-12 51.5,-12 51.5,-6 57.5,0 63.5,0 63.5,0 141.75,0 141.75,0 147.75,0 153.75,-6 153.75,-12 153.75,-12 153.75,-24 153.75,-24 153.75,-30 147.75,-36 141.75,-36\"/>\n",
"<text text-anchor=\"middle\" x=\"102.62\" y=\"-12.95\" font-family=\"Times,serif\" font-size=\"14.00\">generate_image</text>\n",
"</g>\n",
"<!-- decide_mode&#45;&gt;generate_image -->\n",
"<g id=\"edge5\" class=\"edge\">\n",
"<title>decide_mode&#45;&gt;generate_image</title>\n",
"<path fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" d=\"M188.07,-64.62C174.66,-57.46 158.94,-49.07 144.68,-41.45\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"146.55,-37.95 136.08,-36.33 143.25,-44.12 146.55,-37.95\"/>\n",
"</g>\n",
"<!-- generate_code -->\n",
"<g id=\"node7\" class=\"node\">\n",
"<title>generate_code</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M253.62,-36C253.62,-36 183.62,-36 183.62,-36 177.62,-36 171.62,-30 171.62,-24 171.62,-24 171.62,-12 171.62,-12 171.62,-6 177.62,0 183.62,0 183.62,0 253.62,0 253.62,0 259.62,0 265.62,-6 265.62,-12 265.62,-12 265.62,-24 265.62,-24 265.62,-30 259.62,-36 253.62,-36\"/>\n",
"<text text-anchor=\"middle\" x=\"218.62\" y=\"-12.95\" font-family=\"Times,serif\" font-size=\"14.00\">generate_code</text>\n",
"</g>\n",
"<!-- decide_mode&#45;&gt;generate_code -->\n",
"<g id=\"edge6\" class=\"edge\">\n",
"<title>decide_mode&#45;&gt;generate_code</title>\n",
"<path fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" d=\"M220.08,-64.78C219.9,-59.23 219.7,-52.92 219.51,-46.8\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"222.99,-47.08 219.17,-37.19 215.99,-47.3 222.99,-47.08\"/>\n",
"</g>\n",
"<!-- answer_question -->\n",
"<g id=\"node8\" class=\"node\">\n",
"<title>answer_question</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M378,-36C378,-36 295.25,-36 295.25,-36 289.25,-36 283.25,-30 283.25,-24 283.25,-24 283.25,-12 283.25,-12 283.25,-6 289.25,0 295.25,0 295.25,0 378,0 378,0 384,0 390,-6 390,-12 390,-12 390,-24 390,-24 390,-30 384,-36 378,-36\"/>\n",
"<text text-anchor=\"middle\" x=\"336.62\" y=\"-12.95\" font-family=\"Times,serif\" font-size=\"14.00\">answer_question</text>\n",
"</g>\n",
"<!-- decide_mode&#45;&gt;answer_question -->\n",
"<g id=\"edge7\" class=\"edge\">\n",
"<title>decide_mode&#45;&gt;answer_question</title>\n",
"<path fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" d=\"M252.63,-64.62C265.82,-57.46 281.26,-49.07 295.29,-41.45\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"296.61,-44.17 303.72,-36.33 293.27,-38.02 296.61,-44.17\"/>\n",
"</g>\n",
"<!-- prompt_for_more -->\n",
"<g id=\"node9\" class=\"node\">\n",
"<title>prompt_for_more</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M509.38,-36C509.38,-36 419.88,-36 419.88,-36 413.88,-36 407.88,-30 407.88,-24 407.88,-24 407.88,-12 407.88,-12 407.88,-6 413.88,0 419.88,0 419.88,0 509.38,0 509.38,0 515.38,0 521.38,-6 521.38,-12 521.38,-12 521.38,-24 521.38,-24 521.38,-30 515.38,-36 509.38,-36\"/>\n",
"<text text-anchor=\"middle\" x=\"464.62\" y=\"-12.95\" font-family=\"Times,serif\" font-size=\"14.00\">prompt_for_more</text>\n",
"</g>\n",
"<!-- decide_mode&#45;&gt;prompt_for_more -->\n",
"<g id=\"edge8\" class=\"edge\">\n",
"<title>decide_mode&#45;&gt;prompt_for_more</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M265.38,-70.44C302.29,-60.92 355.22,-47.25 397.25,-36.4\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"397.88,-39.59 406.68,-33.7 396.13,-32.81 397.88,-39.59\"/>\n",
"</g>\n",
"<!-- generate_image&#45;&gt;prompt -->\n",
"<g id=\"edge9\" class=\"edge\">\n",
"<title>generate_image&#45;&gt;prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M113.91,-36.29C135.98,-70.09 184.97,-145.13 211.22,-185.34\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"208.82,-187.07 217.22,-193.53 214.69,-183.25 208.82,-187.07\"/>\n",
"</g>\n",
"<!-- generate_code&#45;&gt;prompt -->\n",
"<g id=\"edge11\" class=\"edge\">\n",
"<title>generate_code&#45;&gt;prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M248.55,-36.42C258.43,-43.86 268.25,-53.5 273.62,-65 292.49,-105.36 267.62,-155.59 248.03,-185.66\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"244.63,-183.43 241.91,-193.67 250.42,-187.36 244.63,-183.43\"/>\n",
"</g>\n",
"<!-- answer_question&#45;&gt;prompt -->\n",
"<g id=\"edge10\" class=\"edge\">\n",
"<title>answer_question&#45;&gt;prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M333.13,-36.16C326.64,-65 310.69,-124.15 281.62,-166 276.06,-174.01 268.8,-181.51 261.44,-188.03\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"259.9,-185.6 254.49,-194.71 264.4,-190.96 259.9,-185.6\"/>\n",
"</g>\n",
"<!-- prompt_for_more&#45;&gt;prompt -->\n",
"<g id=\"edge12\" class=\"edge\">\n",
"<title>prompt_for_more&#45;&gt;prompt</title>\n",
"<path fill=\"none\" stroke=\"black\" d=\"M448.47,-36.1C420.81,-64.86 361.91,-123.88 305.62,-166 293.4,-175.14 279.25,-184.03 266.44,-191.51\"/>\n",
"<polygon fill=\"black\" stroke=\"black\" points=\"264.9,-188.94 257.96,-196.94 268.38,-195.01 264.9,-188.94\"/>\n",
"</g>\n",
"</g>\n",
"</svg>\n"
],
"text/plain": [
"<graphviz.graphs.Digraph at 0x10833dc50>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"app.visualize()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "f6b7a4cd-a7f8-45db-952c-2cbf0f88e1ae",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"all terminal actions: ['generate_image', 'answer_question', 'generate_code', 'prompt_for_more', 'unsafe_response']\n"
]
}
],
"source": [
"print(\"all terminal actions: \", TERMINAL_ACTIONS)"
]
},
{
"cell_type": "markdown",
"id": "6f57c117-8951-4f98-86eb-ea1eac16347b",
"metadata": {},
"source": [
"# Calling the application\n",
"\n",
"With streaming, we get back a `StreamingResultContainer`. This allows us to get partial results, while also allowing us to get the full result.\n",
"In the following case, we just "
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "eeb23b59-e207-47b1-b500-b60d431396f8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"I\n",
" can\n",
" generate\n",
" a\n",
" long\n",
" sentence\n",
" for\n",
" you\n",
",\n",
" but\n",
" keep\n",
" in\n",
" mind\n",
" that\n",
" it\n",
" may\n",
" not\n",
" be\n",
" perfect\n",
" in\n",
" terms\n",
" of\n",
" literary\n",
" quality\n",
" or\n",
" coherence\n",
":\n",
" \n",
"\n",
"\n",
"\"In\n",
" the\n",
" lush\n",
",\n",
" verd\n",
"ant\n",
" me\n",
"adow\n",
",\n",
" the\n",
" golden\n",
" sun\n",
" cast\n",
" its\n",
" warm\n",
" rays\n",
" upon\n",
" the\n",
" vibrant\n",
" flowers\n",
",\n",
" painting\n",
" a\n",
" stunning\n",
" kale\n",
"idos\n",
"cope\n",
" of\n",
" colors\n",
" that\n",
" danced\n",
" in\n",
" the\n",
" gentle\n",
" breeze\n",
",\n",
" while\n",
" the\n",
" melod\n",
"ious\n",
" chir\n",
"ping\n",
" of\n",
" birds\n",
" filled\n",
" the\n",
" air\n",
" with\n",
" a\n",
" sweet\n",
" sym\n",
"phony\n",
" that\n",
" lifted\n",
" the\n",
" spirits\n",
" of\n",
" all\n",
" who\n",
" b\n",
"ask\n",
"ed\n",
" in\n",
" the\n",
" beauty\n",
" of\n",
" nature\n",
"'s\n",
" embrace\n",
".\"\n",
"({'response': {'content': 'I can generate a long sentence for you, but keep in mind that it may not be perfect in terms of literary quality or coherence: \\n\\n\"In the lush, verdant meadow, the golden sun cast its warm rays upon the vibrant flowers, painting a stunning kaleidoscope of colors that danced in the gentle breeze, while the melodious chirping of birds filled the air with a sweet symphony that lifted the spirits of all who basked in the beauty of nature\\'s embrace.\"', 'type': 'text', 'role': 'assistant'}}, {'chat_history': [{'role': 'user', 'content': 'Please answer the following question:: please write a long poem', 'type': 'text'}, {'content': \"In the depths of darkness, I search for light\\nEyes wide open, scanning for a glimmer\\nThrough the shadows, my soul takes flight\\nHoping for a spark to become a shimmer\\n\\nBut the night is deep and the stars are hidden\\nI wander aimlessly, lost in the vast unknown\\nThe silence surrounds me, my heart is ridden\\nWith fear and doubt, my spirit is prone\\n\\nYet still I press on, with courage in my heart\\nKnowing that somewhere, there is a guiding star\\nI follow its faint glow, clinging to hope's art\\nTrusting that it will lead me to a place afar\\n\\nThrough the trials and tribulations, I endure\\nEach step a challenge, each moment a test\\nBut with unwavering faith, I am sure\\nThat I will find my way, I will find rest\\n\\nAnd finally, as the first rays of dawn break\\nI see the light, shining bright and clear\\nIn its warmth, all my fears and doubts forsake\\nAnd I know, I am safe, I am near\\n\\nThe journey was long, the path was steep\\nBut in the end, it was all worth the fight\\nFor in the darkness, I found the light to keep\\nAnd emerged stronger, in the day's first light.\", 'type': 'text', 'role': 'assistant'}, {'role': 'user', 'content': 'please write a long poem', 'type': 'text'}, {'content': 'None of the response modes I support apply to your question. Please clarify?', 'type': 'text', 'role': 'assistant'}, {'role': 'user', 'content': 'please write a long poem', 'type': 'text'}, {'content': 'None of the response modes I support apply to your question. Please clarify?', 'type': 'text', 'role': 'assistant'}, {'role': 'user', 'content': \"Please answer the following question:: What is the longest sentence you can think of that isn't a run-on?\", 'type': 'text'}, {'content': 'I can generate a long sentence for you, but keep in mind that it may not be perfect in terms of literary quality or coherence: \\n\\n\"In the lush, verdant meadow, the golden sun cast its warm rays upon the vibrant flowers, painting a stunning kaleidoscope of colors that danced in the gentle breeze, while the melodious chirping of birds filled the air with a sweet symphony that lifted the spirits of all who basked in the beauty of nature\\'s embrace.\"', 'type': 'text', 'role': 'assistant'}], '__SEQUENCE_ID': 15, 'prompt': \"What is the longest sentence you can think of that isn't a run-on?\", '__PRIOR_STEP': 'answer_question', 'safe': True, 'mode': 'answer_question', 'response': {'content': 'I can generate a long sentence for you, but keep in mind that it may not be perfect in terms of literary quality or coherence: \\n\\n\"In the lush, verdant meadow, the golden sun cast its warm rays upon the vibrant flowers, painting a stunning kaleidoscope of colors that danced in the gentle breeze, while the melodious chirping of birds filled the air with a sweet symphony that lifted the spirits of all who basked in the beauty of nature\\'s embrace.\"', 'type': 'text', 'role': 'assistant'}})\n"
]
}
],
"source": [
"action, streaming_container = app.stream_result(\n",
" halt_after=TERMINAL_ACTIONS, inputs={\"prompt\": \"What is the longest sentence you can think of that isn't a run-on?\"}\n",
")\n",
"if action.streaming:\n",
" for item in streaming_container:\n",
" print(item['response']['content'])\n",
" print(streaming_container.get())\n",
"else:\n",
" print(streaming_container.get()[0][\"response\"])"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}