images/bert-pytorch/bert-tvm_68_0.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.43.0 (0)
 -->
<!-- Title: %3 Pages: 1 -->
<svg width="2122pt" height="1916pt"
 viewBox="0.00 0.00 2122.14 1916.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1912)">
<title>%3</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-1912 2118.14,-1912 2118.14,4 -4,4"/>
<!-- 0 -->
<g id="node1" class="node">
<title>0</title>
<ellipse fill="none" stroke="black" cx="1461.64" cy="-1746" rx="170.87" ry="18"/>
<text text-anchor="middle" x="1461.64" y="-1742.3" font-family="Times,serif" font-size="14.00">input: Tensor[(1, 14, 768), float32]</text>
</g>
<!-- 16 -->
<g id="node9" class="node">
<title>16</title>
<polygon fill="none" stroke="black" points="1620.64,-1692 1268.64,-1692 1268.64,-1656 1620.64,-1656 1620.64,-1692"/>
<text text-anchor="middle" x="1444.64" y="-1670.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 14, 768], reverse=0)</text>
</g>
<!-- 0&#45;&gt;16 -->
<g id="edge1" class="edge">
<title>0&#45;&gt;16</title>
<path fill="none" stroke="black" d="M1457.43,-1727.7C1455.54,-1719.9 1453.26,-1710.51 1451.15,-1701.83"/>
<polygon fill="black" stroke="black" points="1454.55,-1701 1448.79,-1692.1 1447.75,-1702.65 1454.55,-1701"/>
</g>
<!-- 1 -->
<g id="node2" class="node">
<title>1</title>
<ellipse fill="none" stroke="black" cx="1092.64" cy="-1890" rx="200.36" ry="18"/>
<text text-anchor="middle" x="1092.64" y="-1886.3" font-family="Times,serif" font-size="14.00">query.weight: Tensor[(768, 768), float32]</text>
</g>
<!-- 17 -->
<g id="node10" class="node">
<title>17</title>
<polygon fill="none" stroke="black" points="1189.14,-1836 996.14,-1836 996.14,-1800 1189.14,-1800 1189.14,-1836"/>
<text text-anchor="middle" x="1092.64" y="-1814.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[1, 0])</text>
</g>
<!-- 1&#45;&gt;17 -->
<g id="edge2" class="edge">
<title>1&#45;&gt;17</title>
<path fill="none" stroke="black" d="M1092.64,-1871.7C1092.64,-1863.98 1092.64,-1854.71 1092.64,-1846.11"/>
<polygon fill="black" stroke="black" points="1096.14,-1846.1 1092.64,-1836.1 1089.14,-1846.1 1096.14,-1846.1"/>
</g>
<!-- 2 -->
<g id="node3" class="node">
<title>2</title>
<ellipse fill="none" stroke="black" cx="863.64" cy="-1530" rx="167.07" ry="18"/>
<text text-anchor="middle" x="863.64" y="-1526.3" font-family="Times,serif" font-size="14.00">query.bias: Tensor[(768,), float32]</text>
</g>
<!-- 22 -->
<g id="node15" class="node">
<title>22</title>
<polygon fill="none" stroke="black" points="1002.14,-1476 931.14,-1476 931.14,-1440 1002.14,-1440 1002.14,-1476"/>
<text text-anchor="middle" x="966.64" y="-1454.3" font-family="Times,serif" font-size="14.00">add(·, ·)</text>
</g>
<!-- 2&#45;&gt;22 -->
<g id="edge9" class="edge">
<title>2&#45;&gt;22</title>
<path fill="none" stroke="black" d="M888.57,-1512.05C901.88,-1503.01 918.46,-1491.74 932.9,-1481.93"/>
<polygon fill="black" stroke="black" points="935.27,-1484.55 941.58,-1476.03 931.34,-1478.76 935.27,-1484.55"/>
</g>
<!-- 3 -->
<g id="node4" class="node">
<title>3</title>
<ellipse fill="none" stroke="black" cx="564.64" cy="-1890" rx="189.57" ry="18"/>
<text text-anchor="middle" x="564.64" y="-1886.3" font-family="Times,serif" font-size="14.00">key.weight: Tensor[(768, 768), float32]</text>
</g>
<!-- 26 -->
<g id="node19" class="node">
<title>26</title>
<polygon fill="none" stroke="black" points="661.14,-1836 468.14,-1836 468.14,-1800 661.14,-1800 661.14,-1836"/>
<text text-anchor="middle" x="564.64" y="-1814.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[1, 0])</text>
</g>
<!-- 3&#45;&gt;26 -->
<g id="edge13" class="edge">
<title>3&#45;&gt;26</title>
<path fill="none" stroke="black" d="M564.64,-1871.7C564.64,-1863.98 564.64,-1854.71 564.64,-1846.11"/>
<polygon fill="black" stroke="black" points="568.14,-1846.1 564.64,-1836.1 561.14,-1846.1 568.14,-1846.1"/>
</g>
<!-- 4 -->
<g id="node5" class="node">
<title>4</title>
<ellipse fill="none" stroke="black" cx="156.64" cy="-1530" rx="156.77" ry="18"/>
<text text-anchor="middle" x="156.64" y="-1526.3" font-family="Times,serif" font-size="14.00">key.bias: Tensor[(768,), float32]</text>
</g>
<!-- 31 -->
<g id="node24" class="node">
<title>31</title>
<polygon fill="none" stroke="black" points="540.14,-1476 469.14,-1476 469.14,-1440 540.14,-1440 540.14,-1476"/>
<text text-anchor="middle" x="504.64" y="-1454.3" font-family="Times,serif" font-size="14.00">add(·, ·)</text>
</g>
<!-- 4&#45;&gt;31 -->
<g id="edge20" class="edge">
<title>4&#45;&gt;31</title>
<path fill="none" stroke="black" d="M230.86,-1514.07C299.97,-1500.17 400.8,-1479.89 458.85,-1468.21"/>
<polygon fill="black" stroke="black" points="459.71,-1471.61 468.83,-1466.2 458.33,-1464.74 459.71,-1471.61"/>
</g>
<!-- 5 -->
<g id="node6" class="node">
<title>5</title>
<ellipse fill="none" stroke="black" cx="1325.64" cy="-882" rx="217.96" ry="18"/>
<text text-anchor="middle" x="1325.64" y="-878.3" font-family="Times,serif" font-size="14.00">attention_mask: Tensor[(1, 1, 1, 14), float32]</text>
</g>
<!-- 41 -->
<g id="node33" class="node">
<title>41</title>
<polygon fill="none" stroke="black" points="1361.14,-828 1290.14,-828 1290.14,-792 1361.14,-792 1361.14,-828"/>
<text text-anchor="middle" x="1325.64" y="-806.3" font-family="Times,serif" font-size="14.00">add(·, ·)</text>
</g>
<!-- 5&#45;&gt;41 -->
<g id="edge31" class="edge">
<title>5&#45;&gt;41</title>
<path fill="none" stroke="black" d="M1325.64,-863.7C1325.64,-855.98 1325.64,-846.71 1325.64,-838.11"/>
<polygon fill="black" stroke="black" points="1329.14,-838.1 1325.64,-828.1 1322.14,-838.1 1329.14,-838.1"/>
</g>
<!-- 6 -->
<g id="node7" class="node">
<title>6</title>
<ellipse fill="none" stroke="black" cx="1885.64" cy="-1890" rx="200.36" ry="18"/>
<text text-anchor="middle" x="1885.64" y="-1886.3" font-family="Times,serif" font-size="14.00">value.weight: Tensor[(768, 768), float32]</text>
</g>
<!-- 46 -->
<g id="node38" class="node">
<title>46</title>
<polygon fill="none" stroke="black" points="1982.14,-1836 1789.14,-1836 1789.14,-1800 1982.14,-1800 1982.14,-1836"/>
<text text-anchor="middle" x="1885.64" y="-1814.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[1, 0])</text>
</g>
<!-- 6&#45;&gt;46 -->
<g id="edge36" class="edge">
<title>6&#45;&gt;46</title>
<path fill="none" stroke="black" d="M1885.64,-1871.7C1885.64,-1863.98 1885.64,-1854.71 1885.64,-1846.11"/>
<polygon fill="black" stroke="black" points="1889.14,-1846.1 1885.64,-1836.1 1882.14,-1846.1 1889.14,-1846.1"/>
</g>
<!-- 7 -->
<g id="node8" class="node">
<title>7</title>
<ellipse fill="none" stroke="black" cx="1581.64" cy="-1530" rx="167.07" ry="18"/>
<text text-anchor="middle" x="1581.64" y="-1526.3" font-family="Times,serif" font-size="14.00">value.bias: Tensor[(768,), float32]</text>
</g>
<!-- 51 -->
<g id="node43" class="node">
<title>51</title>
<polygon fill="none" stroke="black" points="1617.14,-1476 1546.14,-1476 1546.14,-1440 1617.14,-1440 1617.14,-1476"/>
<text text-anchor="middle" x="1581.64" y="-1454.3" font-family="Times,serif" font-size="14.00">add(·, ·)</text>
</g>
<!-- 7&#45;&gt;51 -->
<g id="edge43" class="edge">
<title>7&#45;&gt;51</title>
<path fill="none" stroke="black" d="M1581.64,-1511.7C1581.64,-1503.98 1581.64,-1494.71 1581.64,-1486.11"/>
<polygon fill="black" stroke="black" points="1585.14,-1486.1 1581.64,-1476.1 1578.14,-1486.1 1585.14,-1486.1"/>
</g>
<!-- 20 -->
<g id="node13" class="node">
<title>20</title>
<polygon fill="none" stroke="black" points="1307.14,-1620 1138.14,-1620 1138.14,-1584 1307.14,-1584 1307.14,-1620"/>
<text text-anchor="middle" x="1222.64" y="-1598.3" font-family="Times,serif" font-size="14.00">nn.batch_matmul(·, ·)</text>
</g>
<!-- 16&#45;&gt;20 -->
<g id="edge5" class="edge">
<title>16&#45;&gt;20</title>
<path fill="none" stroke="black" d="M1390.61,-1655.97C1359.19,-1646.06 1319.41,-1633.51 1286.45,-1623.12"/>
<polygon fill="black" stroke="black" points="1287.49,-1619.78 1276.9,-1620.11 1285.39,-1626.46 1287.49,-1619.78"/>
</g>
<!-- 29 -->
<g id="node22" class="node">
<title>29</title>
<polygon fill="none" stroke="black" points="649.14,-1620 480.14,-1620 480.14,-1584 649.14,-1584 649.14,-1620"/>
<text text-anchor="middle" x="564.64" y="-1598.3" font-family="Times,serif" font-size="14.00">nn.batch_matmul(·, ·)</text>
</g>
<!-- 16&#45;&gt;29 -->
<g id="edge16" class="edge">
<title>16&#45;&gt;29</title>
<path fill="none" stroke="black" d="M1268.41,-1658.98C1085.06,-1644.4 804.87,-1622.11 659.51,-1610.55"/>
<polygon fill="black" stroke="black" points="659.53,-1607.04 649.28,-1609.73 658.97,-1614.02 659.53,-1607.04"/>
</g>
<!-- 49 -->
<g id="node41" class="node">
<title>49</title>
<polygon fill="none" stroke="black" points="1970.14,-1620 1801.14,-1620 1801.14,-1584 1970.14,-1584 1970.14,-1620"/>
<text text-anchor="middle" x="1885.64" y="-1598.3" font-family="Times,serif" font-size="14.00">nn.batch_matmul(·, ·)</text>
</g>
<!-- 16&#45;&gt;49 -->
<g id="edge39" class="edge">
<title>16&#45;&gt;49</title>
<path fill="none" stroke="black" d="M1551.95,-1655.97C1624.98,-1644.37 1720.72,-1629.18 1790.91,-1618.04"/>
<polygon fill="black" stroke="black" points="1791.75,-1621.45 1801.08,-1616.42 1790.65,-1614.53 1791.75,-1621.45"/>
</g>
<!-- 18 -->
<g id="node11" class="node">
<title>18</title>
<polygon fill="none" stroke="black" points="1273.14,-1764 912.14,-1764 912.14,-1728 1273.14,-1728 1273.14,-1764"/>
<text text-anchor="middle" x="1092.64" y="-1742.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 768, 768], reverse=0)</text>
</g>
<!-- 17&#45;&gt;18 -->
<g id="edge3" class="edge">
<title>17&#45;&gt;18</title>
<path fill="none" stroke="black" d="M1092.64,-1799.7C1092.64,-1791.98 1092.64,-1782.71 1092.64,-1774.11"/>
<polygon fill="black" stroke="black" points="1096.14,-1774.1 1092.64,-1764.1 1089.14,-1774.1 1096.14,-1774.1"/>
</g>
<!-- 19 -->
<g id="node12" class="node">
<title>19</title>
<polygon fill="none" stroke="black" points="1216.14,-1692 1005.14,-1692 1005.14,-1656 1216.14,-1656 1216.14,-1692"/>
<text text-anchor="middle" x="1110.64" y="-1670.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1])</text>
</g>
<!-- 18&#45;&gt;19 -->
<g id="edge4" class="edge">
<title>18&#45;&gt;19</title>
<path fill="none" stroke="black" d="M1097.09,-1727.7C1099.09,-1719.9 1101.51,-1710.51 1103.74,-1701.83"/>
<polygon fill="black" stroke="black" points="1107.14,-1702.66 1106.24,-1692.1 1100.36,-1700.92 1107.14,-1702.66"/>
</g>
<!-- 19&#45;&gt;20 -->
<g id="edge6" class="edge">
<title>19&#45;&gt;20</title>
<path fill="none" stroke="black" d="M1138.03,-1655.88C1152.54,-1646.81 1170.55,-1635.55 1186.22,-1625.76"/>
<polygon fill="black" stroke="black" points="1188.51,-1628.46 1195.13,-1620.19 1184.8,-1622.52 1188.51,-1628.46"/>
</g>
<!-- 21 -->
<g id="node14" class="node">
<title>21</title>
<polygon fill="none" stroke="black" points="1396.14,-1548 1049.14,-1548 1049.14,-1512 1396.14,-1512 1396.14,-1548"/>
<text text-anchor="middle" x="1222.64" y="-1526.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 768], reverse=0)</text>
</g>
<!-- 20&#45;&gt;21 -->
<g id="edge7" class="edge">
<title>20&#45;&gt;21</title>
<path fill="none" stroke="black" d="M1222.64,-1583.7C1222.64,-1575.98 1222.64,-1566.71 1222.64,-1558.11"/>
<polygon fill="black" stroke="black" points="1226.14,-1558.1 1222.64,-1548.1 1219.14,-1558.1 1226.14,-1558.1"/>
</g>
<!-- 21&#45;&gt;22 -->
<g id="edge8" class="edge">
<title>21&#45;&gt;22</title>
<path fill="none" stroke="black" d="M1160.34,-1511.97C1114.53,-1499.44 1053.33,-1482.71 1012.08,-1471.43"/>
<polygon fill="black" stroke="black" points="1012.87,-1468.01 1002.3,-1468.75 1011.03,-1474.77 1012.87,-1468.01"/>
</g>
<!-- 23 -->
<g id="node16" class="node">
<title>23</title>
<polygon fill="none" stroke="black" points="1149.14,-1404 784.14,-1404 784.14,-1368 1149.14,-1368 1149.14,-1404"/>
<text text-anchor="middle" x="966.64" y="-1382.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 12, 64], reverse=0)</text>
</g>
<!-- 22&#45;&gt;23 -->
<g id="edge10" class="edge">
<title>22&#45;&gt;23</title>
<path fill="none" stroke="black" d="M966.64,-1439.7C966.64,-1431.98 966.64,-1422.71 966.64,-1414.11"/>
<polygon fill="black" stroke="black" points="970.14,-1414.1 966.64,-1404.1 963.14,-1414.1 970.14,-1414.1"/>
</g>
<!-- 24 -->
<g id="node17" class="node">
<title>24</title>
<polygon fill="none" stroke="black" points="1081.14,-1332 852.14,-1332 852.14,-1296 1081.14,-1296 1081.14,-1332"/>
<text text-anchor="middle" x="966.64" y="-1310.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1, 3])</text>
</g>
<!-- 23&#45;&gt;24 -->
<g id="edge11" class="edge">
<title>23&#45;&gt;24</title>
<path fill="none" stroke="black" d="M966.64,-1367.7C966.64,-1359.98 966.64,-1350.71 966.64,-1342.11"/>
<polygon fill="black" stroke="black" points="970.14,-1342.1 966.64,-1332.1 963.14,-1342.1 970.14,-1342.1"/>
</g>
<!-- 25 -->
<g id="node18" class="node">
<title>25</title>
<polygon fill="none" stroke="black" points="1138.14,-1116 795.14,-1116 795.14,-1080 1138.14,-1080 1138.14,-1116"/>
<text text-anchor="middle" x="966.64" y="-1094.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 14, 64], reverse=0)</text>
</g>
<!-- 24&#45;&gt;25 -->
<g id="edge12" class="edge">
<title>24&#45;&gt;25</title>
<path fill="none" stroke="black" d="M966.64,-1295.85C966.64,-1258.83 966.64,-1171.18 966.64,-1126.39"/>
<polygon fill="black" stroke="black" points="970.14,-1126.23 966.64,-1116.23 963.14,-1126.23 970.14,-1126.23"/>
</g>
<!-- 37 -->
<g id="node30" class="node">
<title>37</title>
<polygon fill="none" stroke="black" points="1051.14,-1044 882.14,-1044 882.14,-1008 1051.14,-1008 1051.14,-1044"/>
<text text-anchor="middle" x="966.64" y="-1022.3" font-family="Times,serif" font-size="14.00">nn.batch_matmul(·, ·)</text>
</g>
<!-- 25&#45;&gt;37 -->
<g id="edge26" class="edge">
<title>25&#45;&gt;37</title>
<path fill="none" stroke="black" d="M966.64,-1079.7C966.64,-1071.98 966.64,-1062.71 966.64,-1054.11"/>
<polygon fill="black" stroke="black" points="970.14,-1054.1 966.64,-1044.1 963.14,-1054.1 970.14,-1054.1"/>
</g>
<!-- 27 -->
<g id="node20" class="node">
<title>27</title>
<polygon fill="none" stroke="black" points="745.14,-1764 384.14,-1764 384.14,-1728 745.14,-1728 745.14,-1764"/>
<text text-anchor="middle" x="564.64" y="-1742.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 768, 768], reverse=0)</text>
</g>
<!-- 26&#45;&gt;27 -->
<g id="edge14" class="edge">
<title>26&#45;&gt;27</title>
<path fill="none" stroke="black" d="M564.64,-1799.7C564.64,-1791.98 564.64,-1782.71 564.64,-1774.11"/>
<polygon fill="black" stroke="black" points="568.14,-1774.1 564.64,-1764.1 561.14,-1774.1 568.14,-1774.1"/>
</g>
<!-- 28 -->
<g id="node21" class="node">
<title>28</title>
<polygon fill="none" stroke="black" points="670.14,-1692 459.14,-1692 459.14,-1656 670.14,-1656 670.14,-1692"/>
<text text-anchor="middle" x="564.64" y="-1670.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1])</text>
</g>
<!-- 27&#45;&gt;28 -->
<g id="edge15" class="edge">
<title>27&#45;&gt;28</title>
<path fill="none" stroke="black" d="M564.64,-1727.7C564.64,-1719.98 564.64,-1710.71 564.64,-1702.11"/>
<polygon fill="black" stroke="black" points="568.14,-1702.1 564.64,-1692.1 561.14,-1702.1 568.14,-1702.1"/>
</g>
<!-- 28&#45;&gt;29 -->
<g id="edge17" class="edge">
<title>28&#45;&gt;29</title>
<path fill="none" stroke="black" d="M564.64,-1655.7C564.64,-1647.98 564.64,-1638.71 564.64,-1630.11"/>
<polygon fill="black" stroke="black" points="568.14,-1630.1 564.64,-1620.1 561.14,-1630.1 568.14,-1630.1"/>
</g>
<!-- 30 -->
<g id="node23" class="node">
<title>30</title>
<polygon fill="none" stroke="black" points="678.14,-1548 331.14,-1548 331.14,-1512 678.14,-1512 678.14,-1548"/>
<text text-anchor="middle" x="504.64" y="-1526.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 768], reverse=0)</text>
</g>
<!-- 29&#45;&gt;30 -->
<g id="edge18" class="edge">
<title>29&#45;&gt;30</title>
<path fill="none" stroke="black" d="M549.8,-1583.7C542.61,-1575.3 533.84,-1565.07 525.95,-1555.86"/>
<polygon fill="black" stroke="black" points="528.46,-1553.42 519.3,-1548.1 523.15,-1557.97 528.46,-1553.42"/>
</g>
<!-- 30&#45;&gt;31 -->
<g id="edge19" class="edge">
<title>30&#45;&gt;31</title>
<path fill="none" stroke="black" d="M504.64,-1511.7C504.64,-1503.98 504.64,-1494.71 504.64,-1486.11"/>
<polygon fill="black" stroke="black" points="508.14,-1486.1 504.64,-1476.1 501.14,-1486.1 508.14,-1486.1"/>
</g>
<!-- 32 -->
<g id="node25" class="node">
<title>32</title>
<polygon fill="none" stroke="black" points="766.14,-1404 401.14,-1404 401.14,-1368 766.14,-1368 766.14,-1404"/>
<text text-anchor="middle" x="583.64" y="-1382.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 12, 64], reverse=0)</text>
</g>
<!-- 31&#45;&gt;32 -->
<g id="edge21" class="edge">
<title>31&#45;&gt;32</title>
<path fill="none" stroke="black" d="M524.16,-1439.7C534.02,-1430.97 546.12,-1420.24 556.83,-1410.75"/>
<polygon fill="black" stroke="black" points="559.17,-1413.36 564.33,-1404.1 554.53,-1408.12 559.17,-1413.36"/>
</g>
<!-- 33 -->
<g id="node26" class="node">
<title>33</title>
<polygon fill="none" stroke="black" points="704.14,-1332 475.14,-1332 475.14,-1296 704.14,-1296 704.14,-1332"/>
<text text-anchor="middle" x="589.64" y="-1310.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1, 3])</text>
</g>
<!-- 32&#45;&gt;33 -->
<g id="edge22" class="edge">
<title>32&#45;&gt;33</title>
<path fill="none" stroke="black" d="M585.12,-1367.7C585.78,-1359.98 586.58,-1350.71 587.31,-1342.11"/>
<polygon fill="black" stroke="black" points="590.8,-1342.37 588.17,-1332.1 583.83,-1341.77 590.8,-1342.37"/>
</g>
<!-- 34 -->
<g id="node27" class="node">
<title>34</title>
<polygon fill="none" stroke="black" points="748.14,-1260 519.14,-1260 519.14,-1224 748.14,-1224 748.14,-1260"/>
<text text-anchor="middle" x="633.64" y="-1238.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 1, 3, 2])</text>
</g>
<!-- 33&#45;&gt;34 -->
<g id="edge23" class="edge">
<title>33&#45;&gt;34</title>
<path fill="none" stroke="black" d="M600.51,-1295.7C605.63,-1287.56 611.83,-1277.69 617.48,-1268.7"/>
<polygon fill="black" stroke="black" points="620.53,-1270.43 622.88,-1260.1 614.6,-1266.71 620.53,-1270.43"/>
</g>
<!-- 35 -->
<g id="node28" class="node">
<title>35</title>
<polygon fill="none" stroke="black" points="827.14,-1188 484.14,-1188 484.14,-1152 827.14,-1152 827.14,-1188"/>
<text text-anchor="middle" x="655.64" y="-1166.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 64, 14], reverse=0)</text>
</g>
<!-- 34&#45;&gt;35 -->
<g id="edge24" class="edge">
<title>34&#45;&gt;35</title>
<path fill="none" stroke="black" d="M639.07,-1223.7C641.53,-1215.9 644.48,-1206.51 647.2,-1197.83"/>
<polygon fill="black" stroke="black" points="650.6,-1198.69 650.26,-1188.1 643.92,-1196.59 650.6,-1198.69"/>
</g>
<!-- 36 -->
<g id="node29" class="node">
<title>36</title>
<polygon fill="none" stroke="black" points="772.14,-1116 561.14,-1116 561.14,-1080 772.14,-1080 772.14,-1116"/>
<text text-anchor="middle" x="666.64" y="-1094.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1])</text>
</g>
<!-- 35&#45;&gt;36 -->
<g id="edge25" class="edge">
<title>35&#45;&gt;36</title>
<path fill="none" stroke="black" d="M658.36,-1151.7C659.57,-1143.98 661.02,-1134.71 662.38,-1126.11"/>
<polygon fill="black" stroke="black" points="665.85,-1126.53 663.95,-1116.1 658.94,-1125.44 665.85,-1126.53"/>
</g>
<!-- 36&#45;&gt;37 -->
<g id="edge27" class="edge">
<title>36&#45;&gt;37</title>
<path fill="none" stroke="black" d="M739.64,-1079.97C783.19,-1069.8 838.61,-1056.87 883.79,-1046.33"/>
<polygon fill="black" stroke="black" points="884.62,-1049.73 893.56,-1044.05 883.03,-1042.91 884.62,-1049.73"/>
</g>
<!-- 38 -->
<g id="node31" class="node">
<title>38</title>
<polygon fill="none" stroke="black" points="1189.14,-972 824.14,-972 824.14,-936 1189.14,-936 1189.14,-972"/>
<text text-anchor="middle" x="1006.64" y="-950.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 12, 14, 14], reverse=0)</text>
</g>
<!-- 37&#45;&gt;38 -->
<g id="edge28" class="edge">
<title>37&#45;&gt;38</title>
<path fill="none" stroke="black" d="M976.52,-1007.7C981.13,-999.64 986.7,-989.89 991.79,-980.98"/>
<polygon fill="black" stroke="black" points="994.94,-982.52 996.86,-972.1 988.86,-979.05 994.94,-982.52"/>
</g>
<!-- 40 -->
<g id="node32" class="node">
<title>40</title>
<polygon fill="none" stroke="black" points="1080.14,-900 973.14,-900 973.14,-864 1080.14,-864 1080.14,-900"/>
<text text-anchor="middle" x="1026.64" y="-878.3" font-family="Times,serif" font-size="14.00">divide(·, 8.0)</text>
</g>
<!-- 38&#45;&gt;40 -->
<g id="edge29" class="edge">
<title>38&#45;&gt;40</title>
<path fill="none" stroke="black" d="M1011.58,-935.7C1013.81,-927.9 1016.49,-918.51 1018.97,-909.83"/>
<polygon fill="black" stroke="black" points="1022.37,-910.68 1021.75,-900.1 1015.64,-908.76 1022.37,-910.68"/>
</g>
<!-- 40&#45;&gt;41 -->
<g id="edge30" class="edge">
<title>40&#45;&gt;41</title>
<path fill="none" stroke="black" d="M1080.5,-868.27C1086.61,-866.83 1092.76,-865.38 1098.64,-864 1161.45,-849.25 1234.08,-832.31 1279.92,-821.64"/>
<polygon fill="black" stroke="black" points="1280.91,-825 1289.85,-819.33 1279.32,-818.18 1280.91,-825"/>
</g>
<!-- 42 -->
<g id="node34" class="node">
<title>42</title>
<polygon fill="none" stroke="black" points="1420.64,-756 1246.64,-756 1246.64,-720 1420.64,-720 1420.64,-756"/>
<text text-anchor="middle" x="1333.64" y="-734.3" font-family="Times,serif" font-size="14.00">nn.softmax(·| axis=&#45;1)</text>
</g>
<!-- 41&#45;&gt;42 -->
<g id="edge32" class="edge">
<title>41&#45;&gt;42</title>
<path fill="none" stroke="black" d="M1327.61,-791.7C1328.5,-783.98 1329.55,-774.71 1330.54,-766.11"/>
<polygon fill="black" stroke="black" points="1334.02,-766.44 1331.68,-756.1 1327.07,-765.64 1334.02,-766.44"/>
</g>
<!-- 43 -->
<g id="node35" class="node">
<title>43</title>
<polygon fill="none" stroke="black" points="1448.14,-684 1265.14,-684 1265.14,-648 1448.14,-648 1448.14,-684"/>
<text text-anchor="middle" x="1356.64" y="-662.3" font-family="Times,serif" font-size="14.00">nn.dropout(·| rate=0.1)</text>
</g>
<!-- 42&#45;&gt;43 -->
<g id="edge33" class="edge">
<title>42&#45;&gt;43</title>
<path fill="none" stroke="black" d="M1339.32,-719.7C1341.88,-711.9 1344.97,-702.51 1347.82,-693.83"/>
<polygon fill="black" stroke="black" points="1351.22,-694.7 1351.02,-684.1 1344.57,-692.51 1351.22,-694.7"/>
</g>
<!-- 44 -->
<g id="node36" class="node">
<title>44</title>
<polygon fill="none" stroke="black" points="1452.64,-612 1284.64,-612 1284.64,-576 1452.64,-576 1452.64,-612"/>
<text text-anchor="middle" x="1368.64" y="-590.3" font-family="Times,serif" font-size="14.00">TupleGetItem(idx=0)</text>
</g>
<!-- 43&#45;&gt;44 -->
<g id="edge34" class="edge">
<title>43&#45;&gt;44</title>
<path fill="none" stroke="black" d="M1359.6,-647.7C1360.92,-639.98 1362.51,-630.71 1363.99,-622.11"/>
<polygon fill="black" stroke="black" points="1367.46,-622.55 1365.7,-612.1 1360.56,-621.37 1367.46,-622.55"/>
</g>
<!-- 45 -->
<g id="node37" class="node">
<title>45</title>
<polygon fill="none" stroke="black" points="1546.14,-540 1203.14,-540 1203.14,-504 1546.14,-504 1546.14,-540"/>
<text text-anchor="middle" x="1374.64" y="-518.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 14, 14], reverse=0)</text>
</g>
<!-- 44&#45;&gt;45 -->
<g id="edge35" class="edge">
<title>44&#45;&gt;45</title>
<path fill="none" stroke="black" d="M1370.12,-575.7C1370.78,-567.98 1371.58,-558.71 1372.31,-550.11"/>
<polygon fill="black" stroke="black" points="1375.8,-550.37 1373.17,-540.1 1368.83,-549.77 1375.8,-550.37"/>
</g>
<!-- 56 -->
<g id="node48" class="node">
<title>56</title>
<polygon fill="none" stroke="black" points="1558.14,-468 1389.14,-468 1389.14,-432 1558.14,-432 1558.14,-468"/>
<text text-anchor="middle" x="1473.64" y="-446.3" font-family="Times,serif" font-size="14.00">nn.batch_matmul(·, ·)</text>
</g>
<!-- 45&#45;&gt;56 -->
<g id="edge48" class="edge">
<title>45&#45;&gt;56</title>
<path fill="none" stroke="black" d="M1398.85,-503.88C1411.56,-494.89 1427.3,-483.76 1441.06,-474.03"/>
<polygon fill="black" stroke="black" points="1443.18,-476.82 1449.32,-468.19 1439.14,-471.11 1443.18,-476.82"/>
</g>
<!-- 47 -->
<g id="node39" class="node">
<title>47</title>
<polygon fill="none" stroke="black" points="2066.14,-1764 1705.14,-1764 1705.14,-1728 2066.14,-1728 2066.14,-1764"/>
<text text-anchor="middle" x="1885.64" y="-1742.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 768, 768], reverse=0)</text>
</g>
<!-- 46&#45;&gt;47 -->
<g id="edge37" class="edge">
<title>46&#45;&gt;47</title>
<path fill="none" stroke="black" d="M1885.64,-1799.7C1885.64,-1791.98 1885.64,-1782.71 1885.64,-1774.11"/>
<polygon fill="black" stroke="black" points="1889.14,-1774.1 1885.64,-1764.1 1882.14,-1774.1 1889.14,-1774.1"/>
</g>
<!-- 48 -->
<g id="node40" class="node">
<title>48</title>
<polygon fill="none" stroke="black" points="1991.14,-1692 1780.14,-1692 1780.14,-1656 1991.14,-1656 1991.14,-1692"/>
<text text-anchor="middle" x="1885.64" y="-1670.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1])</text>
</g>
<!-- 47&#45;&gt;48 -->
<g id="edge38" class="edge">
<title>47&#45;&gt;48</title>
<path fill="none" stroke="black" d="M1885.64,-1727.7C1885.64,-1719.98 1885.64,-1710.71 1885.64,-1702.11"/>
<polygon fill="black" stroke="black" points="1889.14,-1702.1 1885.64,-1692.1 1882.14,-1702.1 1889.14,-1702.1"/>
</g>
<!-- 48&#45;&gt;49 -->
<g id="edge40" class="edge">
<title>48&#45;&gt;49</title>
<path fill="none" stroke="black" d="M1885.64,-1655.7C1885.64,-1647.98 1885.64,-1638.71 1885.64,-1630.11"/>
<polygon fill="black" stroke="black" points="1889.14,-1630.1 1885.64,-1620.1 1882.14,-1630.1 1889.14,-1630.1"/>
</g>
<!-- 50 -->
<g id="node42" class="node">
<title>50</title>
<polygon fill="none" stroke="black" points="2114.14,-1548 1767.14,-1548 1767.14,-1512 2114.14,-1512 2114.14,-1548"/>
<text text-anchor="middle" x="1940.64" y="-1526.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 768], reverse=0)</text>
</g>
<!-- 49&#45;&gt;50 -->
<g id="edge41" class="edge">
<title>49&#45;&gt;50</title>
<path fill="none" stroke="black" d="M1899.23,-1583.7C1905.76,-1575.39 1913.7,-1565.28 1920.88,-1556.14"/>
<polygon fill="black" stroke="black" points="1923.77,-1558.13 1927.2,-1548.1 1918.27,-1553.81 1923.77,-1558.13"/>
</g>
<!-- 50&#45;&gt;51 -->
<g id="edge42" class="edge">
<title>50&#45;&gt;51</title>
<path fill="none" stroke="black" d="M1853.27,-1511.97C1781.96,-1498.06 1684.02,-1478.96 1627.31,-1467.91"/>
<polygon fill="black" stroke="black" points="1627.7,-1464.42 1617.22,-1465.94 1626.36,-1471.29 1627.7,-1464.42"/>
</g>
<!-- 52 -->
<g id="node44" class="node">
<title>52</title>
<polygon fill="none" stroke="black" points="1764.14,-1404 1399.14,-1404 1399.14,-1368 1764.14,-1368 1764.14,-1404"/>
<text text-anchor="middle" x="1581.64" y="-1382.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 12, 64], reverse=0)</text>
</g>
<!-- 51&#45;&gt;52 -->
<g id="edge44" class="edge">
<title>51&#45;&gt;52</title>
<path fill="none" stroke="black" d="M1581.64,-1439.7C1581.64,-1431.98 1581.64,-1422.71 1581.64,-1414.11"/>
<polygon fill="black" stroke="black" points="1585.14,-1414.1 1581.64,-1404.1 1578.14,-1414.1 1585.14,-1414.1"/>
</g>
<!-- 53 -->
<g id="node45" class="node">
<title>53</title>
<polygon fill="none" stroke="black" points="1695.14,-1332 1466.14,-1332 1466.14,-1296 1695.14,-1296 1695.14,-1332"/>
<text text-anchor="middle" x="1580.64" y="-1310.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1, 3])</text>
</g>
<!-- 52&#45;&gt;53 -->
<g id="edge45" class="edge">
<title>52&#45;&gt;53</title>
<path fill="none" stroke="black" d="M1581.39,-1367.7C1581.28,-1359.98 1581.15,-1350.71 1581.02,-1342.11"/>
<polygon fill="black" stroke="black" points="1584.52,-1342.05 1580.88,-1332.1 1577.52,-1342.15 1584.52,-1342.05"/>
</g>
<!-- 54 -->
<g id="node46" class="node">
<title>54</title>
<polygon fill="none" stroke="black" points="1751.14,-1260 1408.14,-1260 1408.14,-1224 1751.14,-1224 1751.14,-1260"/>
<text text-anchor="middle" x="1579.64" y="-1238.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[&#45;1, 14, 64], reverse=0)</text>
</g>
<!-- 53&#45;&gt;54 -->
<g id="edge46" class="edge">
<title>53&#45;&gt;54</title>
<path fill="none" stroke="black" d="M1580.39,-1295.7C1580.28,-1287.98 1580.15,-1278.71 1580.02,-1270.11"/>
<polygon fill="black" stroke="black" points="1583.52,-1270.05 1579.88,-1260.1 1576.52,-1270.15 1583.52,-1270.05"/>
</g>
<!-- 55 -->
<g id="node47" class="node">
<title>55</title>
<polygon fill="none" stroke="black" points="1682.14,-1044 1471.14,-1044 1471.14,-1008 1682.14,-1008 1682.14,-1044"/>
<text text-anchor="middle" x="1576.64" y="-1022.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1])</text>
</g>
<!-- 54&#45;&gt;55 -->
<g id="edge47" class="edge">
<title>54&#45;&gt;55</title>
<path fill="none" stroke="black" d="M1579.4,-1223.85C1578.88,-1186.83 1577.65,-1099.18 1577.02,-1054.39"/>
<polygon fill="black" stroke="black" points="1580.52,-1054.18 1576.88,-1044.23 1573.52,-1054.28 1580.52,-1054.18"/>
</g>
<!-- 55&#45;&gt;56 -->
<g id="edge49" class="edge">
<title>55&#45;&gt;56</title>
<path fill="none" stroke="black" d="M1576.64,-1007.95C1576.64,-981.29 1576.64,-928.11 1576.64,-883 1576.64,-883 1576.64,-883 1576.64,-593 1576.64,-552.36 1579.06,-537.21 1555.64,-504 1546.9,-491.62 1534.27,-481.42 1521.53,-473.4"/>
<polygon fill="black" stroke="black" points="1523.13,-470.27 1512.74,-468.19 1519.56,-476.3 1523.13,-470.27"/>
</g>
<!-- 57 -->
<g id="node49" class="node">
<title>57</title>
<polygon fill="none" stroke="black" points="1656.14,-396 1291.14,-396 1291.14,-360 1656.14,-360 1656.14,-396"/>
<text text-anchor="middle" x="1473.64" y="-374.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 12, 14, 64], reverse=0)</text>
</g>
<!-- 56&#45;&gt;57 -->
<g id="edge50" class="edge">
<title>56&#45;&gt;57</title>
<path fill="none" stroke="black" d="M1473.64,-431.7C1473.64,-423.98 1473.64,-414.71 1473.64,-406.11"/>
<polygon fill="black" stroke="black" points="1477.14,-406.1 1473.64,-396.1 1470.14,-406.1 1477.14,-406.1"/>
</g>
<!-- 58 -->
<g id="node50" class="node">
<title>58</title>
<polygon fill="none" stroke="black" points="1588.14,-324 1359.14,-324 1359.14,-288 1588.14,-288 1588.14,-324"/>
<text text-anchor="middle" x="1473.64" y="-302.3" font-family="Times,serif" font-size="14.00">transpose(·| axes=[0, 2, 1, 3])</text>
</g>
<!-- 57&#45;&gt;58 -->
<g id="edge51" class="edge">
<title>57&#45;&gt;58</title>
<path fill="none" stroke="black" d="M1473.64,-359.7C1473.64,-351.98 1473.64,-342.71 1473.64,-334.11"/>
<polygon fill="black" stroke="black" points="1477.14,-334.1 1473.64,-324.1 1470.14,-334.1 1477.14,-334.1"/>
</g>
<!-- 59 -->
<g id="node51" class="node">
<title>59</title>
<polygon fill="none" stroke="black" points="1506.14,-252 1441.14,-252 1441.14,-216 1506.14,-216 1506.14,-252"/>
<text text-anchor="middle" x="1473.64" y="-230.3" font-family="Times,serif" font-size="14.00">copy(·)</text>
</g>
<!-- 58&#45;&gt;59 -->
<g id="edge52" class="edge">
<title>58&#45;&gt;59</title>
<path fill="none" stroke="black" d="M1473.64,-287.7C1473.64,-279.98 1473.64,-270.71 1473.64,-262.11"/>
<polygon fill="black" stroke="black" points="1477.14,-262.1 1473.64,-252.1 1470.14,-262.1 1477.14,-262.1"/>
</g>
<!-- 60 -->
<g id="node52" class="node">
<title>60</title>
<polygon fill="none" stroke="black" points="1647.14,-180 1300.14,-180 1300.14,-144 1647.14,-144 1647.14,-180"/>
<text text-anchor="middle" x="1473.64" y="-158.3" font-family="Times,serif" font-size="14.00">reshape(·| newshape=[1, 14, 768], reverse=0)</text>
</g>
<!-- 59&#45;&gt;60 -->
<g id="edge53" class="edge">
<title>59&#45;&gt;60</title>
<path fill="none" stroke="black" d="M1473.64,-215.7C1473.64,-207.98 1473.64,-198.71 1473.64,-190.11"/>
<polygon fill="black" stroke="black" points="1477.14,-190.1 1473.64,-180.1 1470.14,-190.1 1477.14,-190.1"/>
</g>
<!-- 61 -->
<g id="node53" class="node">
<title>61</title>
<polygon fill="none" stroke="black" points="1516.64,-108 1430.64,-108 1430.64,-72 1516.64,-72 1516.64,-108"/>
<text text-anchor="middle" x="1473.64" y="-86.3" font-family="Times,serif" font-size="14.00">Tuple[...])</text>
</g>
<!-- 60&#45;&gt;61 -->
<g id="edge54" class="edge">
<title>60&#45;&gt;61</title>
<path fill="none" stroke="black" d="M1473.64,-143.7C1473.64,-135.98 1473.64,-126.71 1473.64,-118.11"/>
<polygon fill="black" stroke="black" points="1477.14,-118.1 1473.64,-108.1 1470.14,-118.1 1477.14,-118.1"/>
</g>
<!-- 62 -->
<g id="node54" class="node">
<title>62</title>
<polygon fill="none" stroke="black" points="1513.64,-36 1433.64,-36 1433.64,0 1513.64,0 1513.64,-36"/>
<text text-anchor="middle" x="1473.64" y="-14.3" font-family="Times,serif" font-size="14.00">Function</text>
</g>
<!-- 61&#45;&gt;62 -->
<g id="edge55" class="edge">
<title>61&#45;&gt;62</title>
<path fill="none" stroke="black" d="M1473.64,-71.7C1473.64,-63.98 1473.64,-54.71 1473.64,-46.11"/>
<polygon fill="black" stroke="black" points="1477.14,-46.1 1473.64,-36.1 1470.14,-46.1 1477.14,-46.1"/>
</g>
</g>
</svg>