images/bert-pytorch/bert_layer.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.43.0 (0)
 -->
<!-- Title: %3 Pages: 1 -->
<svg width="1433pt" height="793pt"
 viewBox="0.00 0.00 1432.74 793.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 789)">
<title>%3</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-789 1428.74,-789 1428.74,4 -4,4"/>
<text text-anchor="middle" x="712.37" y="-769.8" font-family="Times,serif" font-size="14.00">BertLayer</text>
<g id="clust1" class="cluster">
<title>cluster_.9</title>
<polygon fill="none" stroke="black" points="63,-208 63,-718 1215,-718 1215,-208 63,-208"/>
<text text-anchor="middle" x="639" y="-702.8" font-family="Times,serif" font-size="14.00">attention (BertAttention)</text>
</g>
<g id="clust2" class="cluster">
<title>cluster_attention..7</title>
<polygon fill="none" stroke="black" points="83,-352 83,-643 907,-643 907,-352 83,-352"/>
<text text-anchor="middle" x="495" y="-627.8" font-family="Times,serif" font-size="14.00">attention.self (BertSelfAttention)</text>
</g>
<!-- inp_1 -->
<g id="node1" class="node">
<title>inp_1</title>
<ellipse fill="none" stroke="black" cx="172" cy="-744" rx="36" ry="18"/>
<text text-anchor="middle" x="172" y="-740.3" font-family="Times,serif" font-size="14.00">inp_1</text>
</g>
<!-- attention.inp_1 -->
<g id="node3" class="node">
<title>attention.inp_1</title>
<ellipse fill="none" stroke="black" cx="172" cy="-669" rx="81.49" ry="18"/>
<text text-anchor="middle" x="172" y="-665.3" font-family="Times,serif" font-size="14.00">attention.inp_1</text>
</g>
<!-- inp_1&#45;&gt;attention.inp_1 -->
<g id="edge1" class="edge">
<title>inp_1&#45;&gt;attention.inp_1</title>
<path fill="none" stroke="black" d="M172,-725.7C172,-717.25 172,-706.87 172,-697.37"/>
<polygon fill="black" stroke="black" points="175.5,-697.18 172,-687.18 168.5,-697.18 175.5,-697.18"/>
</g>
<!-- inp_attention_mask -->
<g id="node2" class="node">
<title>inp_attention_mask</title>
<ellipse fill="none" stroke="black" cx="1324" cy="-669" rx="100.98" ry="18"/>
<text text-anchor="middle" x="1324" y="-665.3" font-family="Times,serif" font-size="14.00">inp_attention_mask</text>
</g>
<!-- attention.inp_attention_mask -->
<g id="node4" class="node">
<title>attention.inp_attention_mask</title>
<ellipse fill="none" stroke="black" cx="1061" cy="-594" rx="146.47" ry="18"/>
<text text-anchor="middle" x="1061" y="-590.3" font-family="Times,serif" font-size="14.00">attention.inp_attention_mask</text>
</g>
<!-- inp_attention_mask&#45;&gt;attention.inp_attention_mask -->
<g id="edge2" class="edge">
<title>inp_attention_mask&#45;&gt;attention.inp_attention_mask</title>
<path fill="none" stroke="black" d="M1272.04,-653.58C1230.48,-642.04 1171.94,-625.79 1127.05,-613.33"/>
<polygon fill="black" stroke="black" points="1127.98,-609.96 1117.4,-610.66 1126.1,-616.7 1127.98,-609.96"/>
</g>
<!-- attention.self.inp_1 -->
<g id="node5" class="node">
<title>attention.self.inp_1</title>
<ellipse fill="none" stroke="black" cx="206" cy="-594" rx="100.98" ry="18"/>
<text text-anchor="middle" x="206" y="-590.3" font-family="Times,serif" font-size="14.00">attention.self.inp_1</text>
</g>
<!-- attention.inp_1&#45;&gt;attention.self.inp_1 -->
<g id="edge3" class="edge">
<title>attention.inp_1&#45;&gt;attention.self.inp_1</title>
<path fill="none" stroke="black" d="M179.89,-651.07C184.02,-642.2 189.17,-631.13 193.81,-621.18"/>
<polygon fill="black" stroke="black" points="196.99,-622.63 198.04,-612.09 190.64,-619.68 196.99,-622.63"/>
</g>
<!-- attention..8 -->
<g id="node12" class="node">
<title>attention..8</title>
<polygon fill="none" stroke="black" points="329,-324 71,-324 71,-288 329,-288 329,-324"/>
<text text-anchor="middle" x="200" y="-302.3" font-family="Times,serif" font-size="14.00">attention.output (BertSelfOutput)</text>
</g>
<!-- attention.inp_1&#45;&gt;attention..8 -->
<g id="edge13" class="edge">
<title>attention.inp_1&#45;&gt;attention..8</title>
<path fill="none" stroke="black" d="M102.14,-659.76C93.24,-655.92 85.12,-650.52 79,-643 45.27,-601.57 72,-576.42 72,-523 72,-523 72,-523 72,-449 72,-405.78 53.07,-386.58 79,-352 86.3,-342.26 95.95,-334.63 106.62,-328.65"/>
<polygon fill="black" stroke="black" points="108.45,-331.65 115.79,-324.01 105.29,-325.4 108.45,-331.65"/>
</g>
<!-- attention.self.inp_attention_mask -->
<g id="node6" class="node">
<title>attention.self.inp_attention_mask</title>
<ellipse fill="none" stroke="black" cx="733" cy="-522" rx="165.97" ry="18"/>
<text text-anchor="middle" x="733" y="-518.3" font-family="Times,serif" font-size="14.00">attention.self.inp_attention_mask</text>
</g>
<!-- attention.inp_attention_mask&#45;&gt;attention.self.inp_attention_mask -->
<g id="edge4" class="edge">
<title>attention.inp_attention_mask&#45;&gt;attention.self.inp_attention_mask</title>
<path fill="none" stroke="black" d="M991.44,-578.15C939.66,-567.1 868.82,-551.99 814.4,-540.37"/>
<polygon fill="black" stroke="black" points="814.95,-536.91 804.44,-538.25 813.49,-543.76 814.95,-536.91"/>
</g>
<!-- attention.self..111 -->
<g id="node7" class="node">
<title>attention.self..111</title>
<polygon fill="none" stroke="black" points="329,-540 111,-540 111,-504 329,-504 329,-540"/>
<text text-anchor="middle" x="220" y="-518.3" font-family="Times,serif" font-size="14.00">attention.self.query (Linear)</text>
</g>
<!-- attention.self.inp_1&#45;&gt;attention.self..111 -->
<g id="edge5" class="edge">
<title>attention.self.inp_1&#45;&gt;attention.self..111</title>
<path fill="none" stroke="black" d="M209.46,-575.7C211,-567.98 212.86,-558.71 214.58,-550.11"/>
<polygon fill="black" stroke="black" points="218.05,-550.6 216.58,-540.1 211.19,-549.22 218.05,-550.6"/>
</g>
<!-- attention.self..112 -->
<g id="node8" class="node">
<title>attention.self..112</title>
<polygon fill="none" stroke="black" points="549,-540 347,-540 347,-504 549,-504 549,-540"/>
<text text-anchor="middle" x="448" y="-518.3" font-family="Times,serif" font-size="14.00">attention.self.key (Linear)</text>
</g>
<!-- attention.self.inp_1&#45;&gt;attention.self..112 -->
<g id="edge6" class="edge">
<title>attention.self.inp_1&#45;&gt;attention.self..112</title>
<path fill="none" stroke="black" d="M256.44,-578.41C292.06,-568.11 340.24,-554.17 379.36,-542.85"/>
<polygon fill="black" stroke="black" points="380.47,-546.18 389.1,-540.04 378.52,-539.45 380.47,-546.18"/>
</g>
<!-- attention.self..113 -->
<g id="node9" class="node">
<title>attention.self..113</title>
<polygon fill="none" stroke="black" points="306.5,-468 91.5,-468 91.5,-432 306.5,-432 306.5,-468"/>
<text text-anchor="middle" x="199" y="-446.3" font-family="Times,serif" font-size="14.00">attention.self.value (Linear)</text>
</g>
<!-- attention.self.inp_1&#45;&gt;attention.self..113 -->
<g id="edge7" class="edge">
<title>attention.self.inp_1&#45;&gt;attention.self..113</title>
<path fill="none" stroke="black" d="M152.56,-578.71C133.17,-570.67 113.28,-558.4 102,-540 93.64,-526.36 93.84,-517.76 102,-504 109.91,-490.66 122.49,-480.47 136,-472.75"/>
<polygon fill="black" stroke="black" points="137.76,-475.79 144.98,-468.04 134.5,-469.59 137.76,-475.79"/>
</g>
<!-- attention.self..114 -->
<g id="node10" class="node">
<title>attention.self..114</title>
<polygon fill="none" stroke="black" points="571,-468 325,-468 325,-432 571,-432 571,-468"/>
<text text-anchor="middle" x="448" y="-446.3" font-family="Times,serif" font-size="14.00">attention.self.dropout (Dropout)</text>
</g>
<!-- attention.self.inp_attention_mask&#45;&gt;attention.self..114 -->
<g id="edge9" class="edge">
<title>attention.self.inp_attention_mask&#45;&gt;attention.self..114</title>
<path fill="none" stroke="black" d="M669.4,-505.38C627.45,-495.08 572.19,-481.5 527.38,-470.5"/>
<polygon fill="black" stroke="black" points="527.94,-467.03 517.4,-468.04 526.27,-473.83 527.94,-467.03"/>
</g>
<!-- attention.self..111&#45;&gt;attention.self..114 -->
<g id="edge10" class="edge">
<title>attention.self..111&#45;&gt;attention.self..114</title>
<path fill="none" stroke="black" d="M275.48,-503.97C307.76,-494.06 348.61,-481.51 382.46,-471.12"/>
<polygon fill="black" stroke="black" points="383.74,-474.39 392.27,-468.11 381.68,-467.7 383.74,-474.39"/>
</g>
<!-- attention.self..112&#45;&gt;attention.self..114 -->
<g id="edge8" class="edge">
<title>attention.self..112&#45;&gt;attention.self..114</title>
<path fill="none" stroke="black" d="M448,-503.7C448,-495.98 448,-486.71 448,-478.11"/>
<polygon fill="black" stroke="black" points="451.5,-478.1 448,-468.1 444.5,-478.1 451.5,-478.1"/>
</g>
<!-- attention.self.out_0 -->
<g id="node11" class="node">
<title>attention.self.out_0</title>
<ellipse fill="none" stroke="black" cx="200" cy="-378" rx="100.98" ry="18"/>
<text text-anchor="middle" x="200" y="-374.3" font-family="Times,serif" font-size="14.00">attention.self.out_0</text>
</g>
<!-- attention.self..113&#45;&gt;attention.self.out_0 -->
<g id="edge11" class="edge">
<title>attention.self..113&#45;&gt;attention.self.out_0</title>
<path fill="none" stroke="black" d="M199.25,-431.7C199.36,-423.98 199.49,-414.71 199.61,-406.11"/>
<polygon fill="black" stroke="black" points="203.11,-406.15 199.76,-396.1 196.11,-406.05 203.11,-406.15"/>
</g>
<!-- attention.self..114&#45;&gt;attention.self.out_0 -->
<g id="edge12" class="edge">
<title>attention.self..114&#45;&gt;attention.self.out_0</title>
<path fill="none" stroke="black" d="M387.65,-431.97C349.35,-421.16 299.93,-407.21 261.43,-396.34"/>
<polygon fill="black" stroke="black" points="261.96,-392.85 251.38,-393.5 260.06,-399.59 261.96,-392.85"/>
</g>
<!-- attention.self.out_0&#45;&gt;attention..8 -->
<g id="edge14" class="edge">
<title>attention.self.out_0&#45;&gt;attention..8</title>
<path fill="none" stroke="black" d="M200,-359.7C200,-351.98 200,-342.71 200,-334.11"/>
<polygon fill="black" stroke="black" points="203.5,-334.1 200,-324.1 196.5,-334.1 203.5,-334.1"/>
</g>
<!-- attention.out_0 -->
<g id="node13" class="node">
<title>attention.out_0</title>
<ellipse fill="none" stroke="black" cx="200" cy="-234" rx="81.49" ry="18"/>
<text text-anchor="middle" x="200" y="-230.3" font-family="Times,serif" font-size="14.00">attention.out_0</text>
</g>
<!-- attention..8&#45;&gt;attention.out_0 -->
<g id="edge15" class="edge">
<title>attention..8&#45;&gt;attention.out_0</title>
<path fill="none" stroke="black" d="M200,-287.7C200,-279.98 200,-270.71 200,-262.11"/>
<polygon fill="black" stroke="black" points="203.5,-262.1 200,-252.1 196.5,-262.1 203.5,-262.1"/>
</g>
<!-- .10 -->
<g id="node14" class="node">
<title>.10</title>
<polygon fill="none" stroke="black" points="248,-180 0,-180 0,-144 248,-144 248,-180"/>
<text text-anchor="middle" x="124" y="-158.3" font-family="Times,serif" font-size="14.00">intermediate (BertIntermediate)</text>
</g>
<!-- attention.out_0&#45;&gt;.10 -->
<g id="edge16" class="edge">
<title>attention.out_0&#45;&gt;.10</title>
<path fill="none" stroke="black" d="M181.99,-216.41C172.45,-207.63 160.57,-196.68 150.04,-186.99"/>
<polygon fill="black" stroke="black" points="152.4,-184.4 142.68,-180.2 147.66,-189.55 152.4,-184.4"/>
</g>
<!-- .11 -->
<g id="node15" class="node">
<title>.11</title>
<polygon fill="none" stroke="black" points="280.5,-108 119.5,-108 119.5,-72 280.5,-72 280.5,-108"/>
<text text-anchor="middle" x="200" y="-86.3" font-family="Times,serif" font-size="14.00">output (BertOutput)</text>
</g>
<!-- attention.out_0&#45;&gt;.11 -->
<g id="edge18" class="edge">
<title>attention.out_0&#45;&gt;.11</title>
<path fill="none" stroke="black" d="M225.39,-216.83C237.38,-207.65 250.47,-194.99 257,-180 263.39,-165.33 263.39,-158.67 257,-144 252.07,-132.68 243.39,-122.68 234.27,-114.51"/>
<polygon fill="black" stroke="black" points="236.5,-111.81 226.58,-108.09 232.01,-117.19 236.5,-111.81"/>
</g>
<!-- .10&#45;&gt;.11 -->
<g id="edge17" class="edge">
<title>.10&#45;&gt;.11</title>
<path fill="none" stroke="black" d="M142.79,-143.7C152.17,-135.05 163.68,-124.45 173.91,-115.03"/>
<polygon fill="black" stroke="black" points="176.45,-117.45 181.43,-108.1 171.7,-112.3 176.45,-117.45"/>
</g>
<!-- out_0 -->
<g id="node16" class="node">
<title>out_0</title>
<ellipse fill="none" stroke="black" cx="200" cy="-18" rx="36.29" ry="18"/>
<text text-anchor="middle" x="200" y="-14.3" font-family="Times,serif" font-size="14.00">out_0</text>
</g>
<!-- .11&#45;&gt;out_0 -->
<g id="edge19" class="edge">
<title>.11&#45;&gt;out_0</title>
<path fill="none" stroke="black" d="M200,-71.7C200,-63.98 200,-54.71 200,-46.11"/>
<polygon fill="black" stroke="black" points="203.5,-46.1 200,-36.1 196.5,-46.1 203.5,-46.1"/>
</g>
</g>
</svg>