| #------------------------------------------------------------- |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| #------------------------------------------------------------- |
| |
| source("scripts/nn/networks/resnet.dml") as resnet |
| source("src/test/scripts/applications/nn/util.dml") as test_util |
| |
| values_test_bottleneck_forward_1 = function() { |
| /* |
| * Testing of values for forward pass of basic block against PyTorch. |
| */ |
| strideh = 1; stridew = 1 |
| C_in = 2; C_base = 2 |
| Hin = 2; Win = 2 |
| N = 3 |
| X = matrix(seq(1, N*C_in*Hin*Win), rows=N, cols=C_in*Hin*Win) |
| W_conv1 = matrix(" 0.5185197 |
| |
| -0.01638347 |
| |
| |
| -0.41337225 |
| |
| -0.27387595", |
| rows=C_base, cols=C_in*1*1) |
| gamma_bn1 = matrix(1, rows=C_base, cols=1) |
| beta_bn1 = matrix(0, rows=C_base, cols=1) |
| W_conv2 = matrix(" -0.20811445 0.15620266 -0.02045412 |
| 0.16239561 0.08874698 -0.04675962 |
| -0.15071772 0.08203228 -0.15123627 |
| |
| -0.05238193 0.14012058 -0.12834892 |
| -0.01153079 -0.03199132 0.02700911 |
| 0.22693978 0.1377839 -0.08849475 |
| |
| |
| 0.20858495 -0.07665969 0.09709726 |
| 0.09753416 -0.00713645 -0.21934068 |
| 0.2215875 -0.05444418 0.16051485 |
| |
| -0.14136882 -0.02405146 0.20206784 |
| -0.20452432 0.00909661 -0.00307493 |
| -0.20145056 -0.0456574 -0.21605067", |
| rows=C_base, cols=C_base*3*3) |
| gamma_bn2 = matrix(1, rows=C_base, cols=1) |
| beta_bn2 = matrix(0, rows=C_base, cols=1) |
| W_conv3 = matrix(" 0.1527785 |
| |
| 0.11908448 |
| |
| |
| 0.40458113 |
| |
| 0.01768601 |
| |
| |
| -0.2847237 |
| |
| -0.25494343 |
| |
| |
| -0.26399058 |
| |
| 0.5500943 |
| |
| |
| 0.22660124 |
| |
| -0.7040346 |
| |
| |
| 0.12641346 |
| |
| -0.09441459 |
| |
| |
| 0.5158523 |
| |
| -0.20682847 |
| |
| |
| 0.61719567 |
| |
| 0.21850073", |
| rows=4*C_base, cols=C_base*1*1) |
| gamma_bn3 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn3 = matrix(0, rows=4*C_base, cols=1) |
| # downsample weights |
| W_conv4 = matrix(" -0.3727211 |
| |
| -0.38662055 |
| |
| |
| 0.42501384 |
| |
| -0.4678393 |
| |
| |
| -0.33239904 |
| |
| 0.38465446 |
| |
| |
| -0.5258211 |
| |
| 0.346785 |
| |
| |
| 0.43058223 |
| |
| 0.19192165 |
| |
| |
| 0.12673676 |
| |
| 0.27335274 |
| |
| |
| 0.5349248 |
| |
| 0.05752403 |
| |
| |
| -0.5090851 |
| |
| 0.6524388", |
| rows=4*C_base, cols=C_in*1*1) |
| gamma_bn4 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| weights = list(W_conv1, gamma_bn1, beta_bn1, W_conv2, gamma_bn2, beta_bn2, |
| W_conv3, gamma_bn3, beta_bn3, W_conv4, gamma_bn4, beta_bn4) |
| mode = "train" |
| |
| ema_mean_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_mean_bn4 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| ema_means_vars = list(ema_mean_bn1, ema_var_bn1, ema_mean_bn2, ema_var_bn2, |
| ema_mean_bn3, ema_var_bn3, ema_mean_bn4, ema_var_bn4) |
| |
| [out, Hout, Wout, ema_means_vars_up, cached_out, cached_means_vars] = resnet::bottleneck_block_forward(X, weights, |
| C_in, C_base, Hin, Win, strideh, stridew, mode, ema_means_vars) |
| |
| Hout_exp = 2 |
| Wout_exp = 2 |
| out_expected = matrix(" 0.6000617 2.6354597 |
| 1.3651271 0.14736581 |
| |
| 0.79244256 3.4513025 |
| 0.5675593 0.33977485 |
| |
| 0. 0. |
| 0. 0. |
| |
| 1.0696087 0. |
| 2.9806733 0.6169146 |
| |
| 0. 0.42738855 |
| 0. 0. |
| |
| 0. 1.3223773 |
| 0. 0. |
| |
| 0. 1.2508607 |
| 0. 0. |
| |
| 0. 0.5346739 |
| 0. 0. |
| |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| 0.19081071 0.62350845 |
| 0.7712998 0.87608457 |
| |
| 0.6735816 0. |
| 0. 0. |
| |
| 0. 0.14371195 |
| 0.28843445 0.3475034 |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| |
| 0. 0.14938474 |
| 0. 0.75312185 |
| |
| 0. 0. |
| 0. 0.5097313 |
| |
| 1.8268178 0. |
| 1.3909308 0. |
| |
| 0. 0.50544345 |
| 0. 0. |
| |
| 1.492898 0. |
| 2.218107 0.67900985 |
| |
| 0.80380654 0.4615401 |
| 2.0888937 2.4350812 |
| |
| 0.58336127 0.9749389 |
| 1.9210864 2.9139411 |
| |
| 0.22877222 2.045148 |
| 1.4386882 3.5448616", |
| rows=N, cols=Hout_exp*Wout_exp*4*C_base) |
| |
| test_util::check_all_close(out, out_expected, 0.00001) |
| } |
| |
| values_test_bottleneck_forward_2 = function() { |
| /* |
| * Testing of values for forward pass of basic block against PyTorch. |
| */ |
| strideh = 2; stridew = 2 |
| C_in = 2; C_base = 4 |
| Hin = 4; Win = 4 |
| N = 3 |
| X = matrix(seq(1, N*C_in*Hin*Win), rows=N, cols=C_in*Hin*Win) |
| W_conv1 = matrix(" 0.54590577 |
| |
| 0.11767608 |
| |
| |
| -0.22960076 |
| |
| 0.43695658 |
| |
| |
| 0.11020315 |
| |
| 0.5713164 |
| |
| |
| 0.07730067 |
| |
| -0.22300252", |
| rows=C_base, cols=C_in*1*1) |
| gamma_bn1 = matrix(1, rows=C_base, cols=1) |
| beta_bn1 = matrix(0, rows=C_base, cols=1) |
| W_conv2 = matrix(" 0.04478061 -0.04519658 0.07014292 |
| 0.1488037 0.09634326 -0.07286209 |
| 0.09621079 0.02982104 0.08463918 |
| |
| -0.10158418 -0.16498475 -0.06439342 |
| -0.12783715 0.1367565 0.04800522 |
| 0.06903559 0.05271019 -0.00289933 |
| |
| 0.1304347 -0.11841893 0.01049395 |
| -0.1137567 0.051392 -0.05739705 |
| 0.05106938 -0.03472358 0.13823198 |
| |
| -0.09878366 -0.09939967 -0.09940567 |
| 0.14990713 0.05554186 0.16037513 |
| -0.13754605 -0.16531269 -0.13039397 |
| |
| |
| -0.11211485 0.06750669 0.05967931 |
| 0.13848741 -0.0860709 -0.11361863 |
| 0.0884297 -0.06736742 0.10115398 |
| |
| -0.03955011 0.09534098 -0.12949467 |
| -0.08410829 0.05081274 0.03523459 |
| -0.04249313 0.09934492 0.11330153 |
| |
| -0.12086223 -0.08897804 0.15261032 |
| -0.05623876 -0.05908607 -0.16126578 |
| -0.09544504 0.04163395 -0.02199887 |
| |
| -0.120981 0.00390945 -0.11384692 |
| -0.14139944 -0.09177711 -0.14586869 |
| -0.10612302 0.1666015 0.03147916 |
| |
| |
| 0.05135995 -0.1554474 -0.10946231 |
| -0.05547597 0.02606185 -0.14665356 |
| -0.07181218 -0.0997781 0.00046188 |
| |
| -0.0620172 -0.01154929 -0.11293828 |
| -0.11439919 -0.09723364 -0.05704957 |
| -0.13154683 0.13974498 -0.03307734 |
| |
| 0.14339946 0.05193035 -0.14113283 |
| 0.1153392 -0.04585747 -0.06388768 |
| -0.13834509 -0.16569345 0.0476851 |
| |
| -0.03640731 0.06488718 -0.13677725 |
| 0.12373818 -0.12234229 -0.0287789 |
| 0.03481162 0.08604197 0.13455172 |
| |
| |
| 0.15182655 -0.13215369 0.04194455 |
| -0.07168766 -0.01826413 -0.12474835 |
| 0.15180977 -0.12232509 0.08907522 |
| |
| 0.05857326 0.05415933 -0.0901077 |
| 0.15149193 0.03662507 0.02144001 |
| -0.1468758 0.06996475 -0.02500343 |
| |
| -0.07635405 0.14315777 0.03715813 |
| -0.09221274 -0.08435649 -0.00795929 |
| 0.09306021 -0.04258897 -0.09509581 |
| |
| -0.05707382 -0.12451248 0.05943875 |
| 0.1290067 -0.15690672 0.03870845 |
| 0.08609863 0.03022157 -0.05935411", |
| rows=C_base, cols=C_base*3*3) |
| gamma_bn2 = matrix(1, rows=C_base, cols=1) |
| beta_bn2 = matrix(0, rows=C_base, cols=1) |
| W_conv3 = matrix(" 0.26097107 |
| |
| 0.2627566 |
| |
| 0.18696362 |
| |
| -0.0878607 |
| |
| |
| -0.13240063 |
| |
| 0.05349046 |
| |
| -0.08832705 |
| |
| -0.14900053 |
| |
| |
| 0.31960344 |
| |
| 0.42969978 |
| |
| -0.04949868 |
| |
| -0.11194843 |
| |
| |
| 0.00729614 |
| |
| -0.02985412 |
| |
| 0.12020564 |
| |
| 0.14011681 |
| |
| |
| -0.45412838 |
| |
| -0.18451887 |
| |
| 0.42106473 |
| |
| 0.19477749 |
| |
| |
| -0.02486879 |
| |
| -0.30145288 |
| |
| -0.30590254 |
| |
| -0.44788343 |
| |
| |
| -0.16298121 |
| |
| 0.16885209 |
| |
| 0.31881082 |
| |
| 0.23084867 |
| |
| |
| -0.44197202 |
| |
| -0.30068123 |
| |
| -0.07890832 |
| |
| 0.48367476 |
| |
| |
| 0.07232875 |
| |
| -0.12948537 |
| |
| 0.20685762 |
| |
| -0.19044077 |
| |
| |
| -0.32362783 |
| |
| 0.36494362 |
| |
| -0.22735089 |
| |
| -0.10023338 |
| |
| |
| -0.49740213 |
| |
| 0.33463532 |
| |
| 0.37881732 |
| |
| 0.1822241 |
| |
| |
| -0.3486371 |
| |
| -0.49346995 |
| |
| -0.40608948 |
| |
| 0.37285012 |
| |
| |
| 0.24005288 |
| |
| 0.42075223 |
| |
| 0.26193494 |
| |
| 0.12654608 |
| |
| |
| -0.00489634 |
| |
| -0.38025302 |
| |
| -0.4283861 |
| |
| -0.46767431 |
| |
| |
| 0.20468098 |
| |
| -0.245484 |
| |
| -0.10062629 |
| |
| -0.28775263 |
| |
| |
| -0.09111178 |
| |
| -0.35191745 |
| |
| -0.32670784 |
| |
| 0.16585541", |
| rows=4*C_base, cols=C_base*1*1) |
| gamma_bn3 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn3 = matrix(0, rows=4*C_base, cols=1) |
| # downsample weights |
| W_conv4 = matrix(" 0.5406104 |
| |
| 0.5869042 |
| |
| |
| -0.16565567 |
| |
| 0.6495562 |
| |
| |
| -0.15492964 |
| |
| 0.14268756 |
| |
| |
| -0.3442585 |
| |
| 0.41527158 |
| |
| |
| 0.62334496 |
| |
| -0.5187534 |
| |
| |
| 0.61461455 |
| |
| 0.13234162 |
| |
| |
| 0.5224168 |
| |
| 0.09576386 |
| |
| |
| 0.34095842 |
| |
| -0.09983712 |
| |
| |
| 0.5450986 |
| |
| 0.10451669 |
| |
| |
| -0.33010566 |
| |
| 0.18024033 |
| |
| |
| -0.32579 |
| |
| -0.08292443 |
| |
| |
| -0.2871973 |
| |
| 0.46907407 |
| |
| |
| -0.558169 |
| |
| -0.32598352 |
| |
| |
| -0.19966906 |
| |
| -0.42516384 |
| |
| |
| 0.06673896 |
| |
| -0.6983946 |
| |
| |
| 0.63859457 |
| |
| -0.60066473", |
| rows=4*C_base, cols=C_in*1*1) |
| gamma_bn4 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| weights = list(W_conv1, gamma_bn1, beta_bn1, W_conv2, gamma_bn2, beta_bn2, |
| W_conv3, gamma_bn3, beta_bn3, W_conv4, gamma_bn4, beta_bn4) |
| mode = "train" |
| |
| ema_mean_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_mean_bn4 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| ema_means_vars = list(ema_mean_bn1, ema_var_bn1, ema_mean_bn2, ema_var_bn2, |
| ema_mean_bn3, ema_var_bn3, ema_mean_bn4, ema_var_bn4) |
| |
| [out, Hout, Wout, ema_means_vars_up, cached_out, cached_means_vars] = resnet::bottleneck_block_forward(X, weights, |
| C_in, C_base, Hin, Win, strideh, stridew, mode, ema_means_vars) |
| |
| Hout_exp = 2 |
| Wout_exp = 2 |
| out_expected = matrix(" 0. 0. |
| 0. 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| 0.9109738 0. |
| 0.26423687 0.02831399 |
| |
| 0. 1.1842504 |
| 0. 0. |
| |
| 0. 0.27886808 |
| 0. 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0.4690894 |
| 0. 0. |
| |
| 0. 0.5754694 |
| 0. 0. |
| |
| 0. 0. |
| 0.15799654 0.06399322 |
| |
| 1.6746848 0.10162199 |
| 1.1494169 0.7138793 |
| |
| 1.7823488 2.4663901 |
| 1.1404462 1.3706592 |
| |
| 0. 0.5186337 |
| 0. 0. |
| |
| 0.46825778 1.802414 |
| 0. 0. |
| |
| 1.8138912 0. |
| 2.061407 1.230821 |
| |
| 1.5826645 0. |
| 1.597543 1.0766873 |
| |
| 0. 0.05367124 |
| 0. 0. |
| |
| |
| 0. 0. |
| 0.14620686 0. |
| |
| 0. 0. |
| 0.5086571 0.7567121 |
| |
| 0. 0. |
| 0. 0. |
| |
| 0.22735572 0.93278456 |
| 0.3514644 0.19731534 |
| |
| 0.36241722 0.7131092 |
| 0.4989733 0.43317872 |
| |
| 0. 0. |
| 0. 0. |
| |
| 0.46766508 0.79504216 |
| 0.88795483 0.6664516 |
| |
| 0.37459964 0.99809396 |
| 0.4362625 0.5768033 |
| |
| 0. 0. |
| 0. 0. |
| |
| 0.67712426 0.07316476 |
| 0.9103337 0.88836694 |
| |
| 0.8823242 0.8489517 |
| 0.75503683 0.45444334 |
| |
| 0.23441726 1.0932865 |
| 0.0098927 0.47565216 |
| |
| 0.2859242 0.03880775 |
| 0.80046034 0.08464086 |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0.8421172 |
| 0. 0.02687168 |
| |
| |
| 2.295794 2.8480763 |
| 2.8238502 2.022955 |
| |
| 1.1745573 0.23529911 |
| 2.7716846 0.32033563 |
| |
| 0.23475814 0.3514259 |
| 0.3389814 0. |
| |
| 0. 0.14817369 |
| 0. 0.9291567 |
| |
| 0. 0. |
| 0.31184435 0.17355764 |
| |
| 1.9307995 2.276723 |
| 1.5024273 2.3755558 |
| |
| 0. 0. |
| 0.94689417 0. |
| |
| 0. 0. |
| 0.16562259 0.579775 |
| |
| 1.3288593 2.229822 |
| 0.2227614 2.0304394 |
| |
| 0. 0. |
| 0.5008893 0. |
| |
| 0. 0. |
| 0. 0. |
| |
| 0. 0. |
| 0. 1.2533097 |
| |
| 0. 0. |
| 0.40518343 0. |
| |
| 0. 0.19982207 |
| 0. 0. |
| |
| 0.03520691 0.69359136 |
| 0. 0.04526865 |
| |
| 0.48097372 1.0040715 |
| 0. 2.744193 ", |
| rows=N, cols=Hout_exp*Wout_exp*4*C_base) |
| |
| test_util::check_all_close(out, out_expected, 0.00001) |
| } |
| |
| values_test_residual_layer_forward = function() { |
| Hin = 3; Win = 3; |
| C_in = 2; C_base = 4; |
| N = 2 |
| blocks = 2 |
| strideh = 2; stridew = 2 |
| mode = "train" |
| |
| X = matrix(seq(1, N*Hin*Win*C_in), rows=N, cols=C_in*Hin*Win) |
| |
| # weights for block 1 |
| W_conv1 = matrix(" 0.64133304 |
| |
| -0.30935785 |
| |
| |
| -0.26633477 |
| |
| 0.70287377 |
| |
| |
| -0.01898092 |
| |
| 0.4187042 |
| |
| |
| 0.26043642 |
| |
| 0.65828437", |
| rows=C_base, cols=C_in*1*1) |
| gamma_bn1 = matrix(1, rows=C_base, cols=1) |
| beta_bn1 = matrix(0, rows=C_base, cols=1) |
| W_conv2 = matrix(" -0.07935387 -0.13985893 0.11053868 |
| -0.1480007 0.1574573 0.15253572 |
| 0.15747602 0.16496314 0.034954 |
| |
| -0.0016536 -0.0961354 0.02648038 |
| 0.11480387 -0.09928016 -0.13879894 |
| 0.06693192 0.13438095 -0.07921901 |
| |
| -0.03791906 0.11089791 0.07005687 |
| 0.16293873 -0.0442756 -0.02163965 |
| 0.0766806 0.03089702 0.05085187 |
| |
| 0.0042686 -0.12960267 0.08154561 |
| 0.01905231 -0.07790075 0.03734766 |
| 0.00651456 0.11704092 0.10655515 |
| |
| |
| -0.07699027 -0.03226709 -0.07629572 |
| 0.15273114 0.10044114 -0.07274394 |
| 0.1516081 0.09493737 0.07546788 |
| |
| -0.0726403 -0.15355454 0.11175342 |
| -0.04025601 -0.03588425 0.12585725 |
| 0.13138913 0.10351892 0.09438397 |
| |
| 0.03881417 -0.12341571 0.02957357 |
| -0.00408512 0.00771086 -0.14772138 |
| -0.02420218 0.02257259 -0.05039396 |
| |
| -0.15493202 0.03471132 -0.07309394 |
| 0.14865221 -0.09820288 -0.01550098 |
| 0.10916193 -0.03161931 -0.08472542 |
| |
| |
| 0.08376767 -0.12832934 -0.00301033 |
| 0.10904421 0.02656408 -0.0239322 |
| 0.04957093 -0.07315841 0.05891012 |
| |
| -0.06879979 -0.12485262 0.03166164 |
| 0.07797651 -0.11737081 -0.01409818 |
| 0.00895227 -0.07557337 0.01686102 |
| |
| -0.06918794 0.12680571 0.05430049 |
| -0.04680433 -0.08028235 -0.0635463 |
| -0.01600344 0.07835184 -0.05069631 |
| |
| -0.07545875 0.07460625 0.03207885 |
| 0.16395922 -0.096992 -0.07280873 |
| 0.0738041 0.08279254 0.02707005 |
| |
| |
| 0.05216871 0.09574054 -0.10493654 |
| -0.13408403 -0.09751024 -0.09929453 |
| -0.13031684 0.06681591 0.13878734 |
| |
| -0.06675949 0.14814068 0.06931506 |
| -0.1507676 -0.15641318 0.02355962 |
| -0.09087996 0.10453825 0.06647408 |
| |
| 0.14790873 0.03328724 0.02318728 |
| -0.04853491 0.1355549 -0.09577837 |
| -0.01273669 0.14791735 0.04074055 |
| |
| 0.14454861 -0.00055562 -0.02485079 |
| -0.0598551 0.1521839 0.13950332 |
| -0.03670852 0.03655088 0.11852266", |
| rows=C_base, cols=C_base*3*3) |
| gamma_bn2 = matrix(1, rows=C_base, cols=1) |
| beta_bn2 = matrix(0, rows=C_base, cols=1) |
| W_conv3 = matrix(" -0.29437304 |
| |
| 0.18341273 |
| |
| -0.4085917 |
| |
| 0.26053882 |
| |
| |
| 0.31638247 |
| |
| 0.4066484 |
| |
| 0.4634807 |
| |
| -0.07265258 |
| |
| |
| 0.24328196 |
| |
| -0.29365987 |
| |
| 0.47914386 |
| |
| -0.48774058 |
| |
| |
| -0.3374917 |
| |
| 0.22080815 |
| |
| -0.038697 |
| |
| -0.28921968 |
| |
| |
| 0.24076295 |
| |
| 0.05936813 |
| |
| -0.37986314 |
| |
| 0.4292264 |
| |
| |
| 0.38059032 |
| |
| -0.2816422 |
| |
| 0.36293274 |
| |
| -0.18708462 |
| |
| |
| -0.3077286 |
| |
| -0.3397568 |
| |
| 0.36514604 |
| |
| -0.49366736 |
| |
| |
| 0.10718703 |
| |
| -0.39724267 |
| |
| -0.47798198 |
| |
| 0.47490656 |
| |
| |
| -0.02514178 |
| |
| -0.09093165 |
| |
| 0.25952834 |
| |
| 0.11998898 |
| |
| |
| 0.36005867 |
| |
| -0.21023494 |
| |
| -0.3787375 |
| |
| -0.0058462 |
| |
| |
| 0.06437212 |
| |
| -0.24833494 |
| |
| -0.10442317 |
| |
| -0.09545577 |
| |
| |
| -0.3191191 |
| |
| 0.13568145 |
| |
| -0.18130076 |
| |
| -0.13757545 |
| |
| |
| 0.13844067 |
| |
| -0.32146406 |
| |
| -0.49979246 |
| |
| -0.13366985 |
| |
| |
| -0.11780888 |
| |
| -0.12696624 |
| |
| 0.2837879 |
| |
| -0.0088647 |
| |
| |
| 0.15086323 |
| |
| 0.02197719 |
| |
| -0.10307682 |
| |
| 0.01073551 |
| |
| |
| 0.21700346 |
| |
| 0.27309185 |
| |
| -0.11225039 |
| |
| 0.21934658", |
| rows=4*C_base, cols=C_base*1*1) |
| gamma_bn3 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn3 = matrix(0, rows=4*C_base, cols=1) |
| # downsample weights |
| W_conv4 = matrix(" -0.46313006 |
| |
| -0.03330129 |
| |
| |
| -0.2322562 |
| |
| 0.6264885 |
| |
| |
| -0.67066795 |
| |
| -0.0445317 |
| |
| |
| 0.68385714 |
| |
| -0.15812147 |
| |
| |
| 0.67979616 |
| |
| 0.1485151 |
| |
| |
| -0.55756646 |
| |
| -0.11412668 |
| |
| |
| -0.2764914 |
| |
| 0.5107109 |
| |
| |
| -0.02068317 |
| |
| 0.09800112 |
| |
| |
| 0.376929 |
| |
| 0.5911364 |
| |
| |
| 0.1532327 |
| |
| -0.66188455 |
| |
| |
| -0.50742537 |
| |
| 0.4692853 |
| |
| |
| -0.40435696 |
| |
| -0.1191256 |
| |
| |
| 0.28702432 |
| |
| 0.59869534 |
| |
| |
| -0.31104982 |
| |
| -0.02253169 |
| |
| |
| 0.2543087 |
| |
| 0.27674532 |
| |
| |
| -0.23902792 |
| |
| 0.4872386 ", |
| rows=4*C_base, cols=C_in*1*1) |
| gamma_bn4 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| weights_block1 = list(W_conv1, gamma_bn1, beta_bn1, W_conv2, gamma_bn2, beta_bn2, |
| W_conv3, gamma_bn3, beta_bn3, W_conv4, gamma_bn4, beta_bn4) |
| mode = "train" |
| |
| ema_mean_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_mean_bn4 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| emas_block1 = list(ema_mean_bn1, ema_var_bn1, ema_mean_bn2, ema_var_bn2, |
| ema_mean_bn3, ema_var_bn3, ema_mean_bn4, ema_var_bn4) |
| |
| # Weights for block 2 |
| C_in = 4*C_base |
| W_conv1_2 = matrix(" -0.15873042 |
| |
| -0.12685311 |
| |
| 0.24818534 |
| |
| -0.11151049 |
| |
| -0.21107942 |
| |
| 0.1189599 |
| |
| 0.17549959 |
| |
| 0.22624752 |
| |
| -0.2047379 |
| |
| -0.11143267 |
| |
| 0.10603568 |
| |
| 0.16207412 |
| |
| 0.16275376 |
| |
| -0.1806739 |
| |
| -0.02120981 |
| |
| -0.0203568 |
| |
| |
| 0.04928717 |
| |
| 0.11670396 |
| |
| 0.23074868 |
| |
| 0.21299681 |
| |
| 0.00101537 |
| |
| 0.01027396 |
| |
| 0.02288854 |
| |
| -0.16491142 |
| |
| -0.1332432 |
| |
| -0.1547856 |
| |
| -0.06269354 |
| |
| -0.11677879 |
| |
| 0.0370577 |
| |
| 0.20312098 |
| |
| 0.04698643 |
| |
| -0.09891671 |
| |
| |
| -0.08488309 |
| |
| -0.11952144 |
| |
| 0.06800526 |
| |
| 0.0516184 |
| |
| -0.21355331 |
| |
| 0.13793284 |
| |
| 0.13408375 |
| |
| 0.06124464 |
| |
| 0.18330386 |
| |
| -0.15059847 |
| |
| 0.24269676 |
| |
| -0.22168538 |
| |
| 0.10450107 |
| |
| 0.02138337 |
| |
| -0.04721311 |
| |
| 0.14241049 |
| |
| |
| 0.08129939 |
| |
| -0.04329139 |
| |
| -0.09548986 |
| |
| -0.08126539 |
| |
| -0.0406346 |
| |
| 0.0537217 |
| |
| 0.00775704 |
| |
| -0.24578935 |
| |
| 0.12068957 |
| |
| -0.20760396 |
| |
| -0.22163606 |
| |
| -0.21903417 |
| |
| -0.17546788 |
| |
| -0.11092317 |
| |
| -0.2458466 |
| |
| -0.11614364", |
| rows=C_base, cols=C_in*1*1) |
| gamma_bn1_2 = matrix(1, rows=C_base, cols=1) |
| beta_bn1_2 = matrix(0, rows=C_base, cols=1) |
| W_conv2_2 = matrix(" -6.49780855e-02 -1.53550386e-01 1.27520934e-01 |
| -1.14963055e-01 3.47854644e-02 -9.77773294e-02 |
| 1.39712289e-01 3.93924862e-02 -1.08044863e-01 |
| |
| -8.77258033e-02 9.06148702e-02 -1.23133540e-01 |
| -1.61552995e-01 -7.69458413e-02 6.84276223e-04 |
| 1.16330996e-01 -5.43563366e-02 5.16963005e-02 |
| |
| -1.61242843e-01 4.86649424e-02 9.66079682e-02 |
| 5.86722940e-02 2.77588516e-02 1.50955319e-02 |
| -1.50006920e-01 3.65351588e-02 -4.17252779e-02 |
| |
| 6.55835271e-02 -8.94589424e-02 -8.49471986e-03 |
| -5.53610176e-02 9.70451981e-02 7.06677288e-02 |
| 1.65115044e-01 3.72097939e-02 -5.02234101e-02 |
| |
| |
| -8.07390213e-02 -4.82785702e-02 6.31745458e-02 |
| -7.80546665e-02 -3.55781764e-02 1.22384578e-02 |
| -5.75916991e-02 1.36988893e-01 9.28030759e-02 |
| |
| -1.29722685e-01 -8.44041556e-02 1.04285926e-02 |
| 1.21865556e-01 -2.04753429e-02 -7.18220100e-02 |
| -3.05865854e-02 -1.00977302e-01 -1.63163647e-01 |
| |
| -2.82911062e-02 1.47659853e-01 3.42025906e-02 |
| 8.68119150e-02 1.07737646e-01 -2.95802504e-02 |
| -7.16226920e-02 -1.23587213e-01 1.40657499e-01 |
| |
| 6.51463717e-02 -1.29032552e-01 -1.39336169e-01 |
| -5.93275055e-02 8.56822282e-02 -1.42511040e-01 |
| -7.40398169e-02 -5.41220903e-02 -4.33833897e-03 |
| |
| |
| -1.18034229e-01 -7.63256997e-02 7.52437115e-03 |
| 1.24137387e-01 -1.49110124e-01 -7.26501942e-02 |
| 9.58135277e-02 1.59387156e-01 2.99520344e-02 |
| |
| 1.51494995e-01 -1.38253689e-01 -1.04445815e-01 |
| -1.05174027e-01 1.84758753e-02 9.58175957e-03 |
| 4.42418605e-02 -3.34148854e-02 6.24543428e-03 |
| |
| -1.62895441e-01 2.09720731e-02 -5.38072586e-02 |
| 1.08301952e-01 -1.19824275e-01 3.94731760e-05 |
| -1.18747398e-01 -1.63983807e-01 7.29161650e-02 |
| |
| 9.75044221e-02 -1.01447850e-02 -1.66090474e-01 |
| 1.39794812e-01 1.51129767e-01 -1.42522335e-01 |
| 2.78563499e-02 -1.24729119e-01 -5.18599004e-02 |
| |
| |
| -1.52138442e-01 -5.16706333e-02 6.49023205e-02 |
| -9.79826078e-02 1.14026591e-01 -5.25996909e-02 |
| -8.55211243e-02 1.02774397e-01 1.27465829e-01 |
| |
| 5.07049710e-02 5.54278344e-02 5.20297140e-02 |
| -3.19346040e-02 1.56156614e-01 4.14624810e-04 |
| -1.10439107e-01 -3.79917622e-02 7.33491927e-02 |
| |
| 9.80907828e-02 1.14526585e-01 6.30494207e-02 |
| -6.73903525e-03 -2.50494480e-03 1.09999135e-01 |
| 1.50500730e-01 1.41846985e-02 6.23831153e-02 |
| |
| -2.22751051e-02 -9.66079682e-02 -6.04704842e-02 |
| 4.52399105e-02 3.01361084e-02 8.97199064e-02 |
| 4.74042743e-02 -9.87502337e-02 -1.09392643e-01", |
| rows=C_base, cols=C_base*3*3) |
| gamma_bn2_2 = matrix(1, rows=C_base, cols=1) |
| beta_bn2_2 = matrix(0, rows=C_base, cols=1) |
| W_conv3_2 = matrix(" 0.06731683 |
| |
| -0.24978322 |
| |
| -0.00666493 |
| |
| 0.08701718 |
| |
| |
| 0.01926881 |
| |
| 0.14511615 |
| |
| -0.01574725 |
| |
| 0.39710528 |
| |
| |
| -0.35157645 |
| |
| 0.48869908 |
| |
| 0.3519063 |
| |
| -0.21306509 |
| |
| |
| 0.40004694 |
| |
| -0.00693274 |
| |
| -0.43690896 |
| |
| 0.46575224 |
| |
| |
| -0.3386495 |
| |
| 0.28560823 |
| |
| -0.11992186 |
| |
| -0.3841107 |
| |
| |
| 0.02532202 |
| |
| -0.03704441 |
| |
| 0.27476293 |
| |
| 0.2742144 |
| |
| |
| -0.28822362 |
| |
| 0.2183528 |
| |
| 0.0658052 |
| |
| -0.20464629 |
| |
| |
| 0.2228418 |
| |
| 0.35666758 |
| |
| -0.01196289 |
| |
| -0.27206528 |
| |
| |
| 0.49370784 |
| |
| -0.37761664 |
| |
| -0.4462306 |
| |
| 0.47271717 |
| |
| |
| -0.38218856 |
| |
| 0.06360751 |
| |
| -0.33563244 |
| |
| -0.04947144 |
| |
| |
| -0.01227283 |
| |
| 0.16202927 |
| |
| 0.27922666 |
| |
| 0.08569479 |
| |
| |
| 0.33986813 |
| |
| -0.14089179 |
| |
| -0.38342243 |
| |
| 0.43166125 |
| |
| |
| -0.15443969 |
| |
| 0.21090645 |
| |
| 0.4235263 |
| |
| -0.3934964 |
| |
| |
| -0.00940305 |
| |
| -0.2315566 |
| |
| -0.4138875 |
| |
| -0.11506236 |
| |
| |
| 0.23220491 |
| |
| -0.4435383 |
| |
| 0.44961244 |
| |
| -0.03749442 |
| |
| |
| 0.14980793 |
| |
| -0.3917358 |
| |
| 0.45487362 |
| |
| 0.35936302", |
| rows=4*C_base, cols=C_base*1*1) |
| gamma_bn3_2 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn3_2 = matrix(0, rows=4*C_base, cols=1) |
| |
| weights_block2 = list(W_conv1_2, gamma_bn1_2, beta_bn1_2, W_conv2_2, gamma_bn2_2, beta_bn2_2, |
| W_conv3_2, gamma_bn3_2, beta_bn3_2) |
| mode = "train" |
| |
| ema_mean_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_mean_bn4 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| emas_block2 = list(ema_mean_bn1, ema_var_bn1, ema_mean_bn2, ema_var_bn2, |
| ema_mean_bn3, ema_var_bn3, ema_mean_bn4, ema_var_bn4) |
| |
| expected_Hout = 2 |
| expected_Wout = 2 |
| expected_out = matrix(" 1.3795955 1.667117 |
| 2.1051257 0.4751625 |
| |
| 0. 0. |
| 2.196381 0. |
| |
| 1.6621143 0.62258995 |
| 0. 1.9935839 |
| |
| 0. 0.563171 |
| 2.068542 0. |
| |
| 0.4887283 0. |
| 0. 0. |
| |
| 0.6661688 0.70850194 |
| 2.4760737 2.0852184 |
| |
| 0.24420133 0. |
| 0. 0.7932879 |
| |
| 0. 0.61385596 |
| 0. 0. |
| |
| 0. 0.82111585 |
| 2.0687509 0. |
| |
| 1.6963062 0. |
| 0. 0. |
| |
| 0.8057394 0.61964965 |
| 0.89105517 2.5202808 |
| |
| 1.6423799 2.2449687 |
| 3.336556 0. |
| |
| 0. 0. |
| 0. 1.5585338 |
| |
| 3.2802167 2.9148674 |
| 1.8678657 0. |
| |
| 0. 0.78625363 |
| 0.49233294 1.973973 |
| |
| 0. 0.11497754 |
| 1.5337045 1.4560605 |
| |
| |
| 0. 0.5596464 |
| 0. 0. |
| |
| 1.0924859 3.3933737 |
| 0.20856619 0.7022976 |
| |
| 1.4003831 0. |
| 0.45457155 0.6552094 |
| |
| 0. 0.6216905 |
| 1.102113 2.0085845 |
| |
| 3.6501603 0.8336022 |
| 2.219977 2.0842493 |
| |
| 0.76798964 1.2647653 |
| 0. 0. |
| |
| 1.0118731 0. |
| 1.9565775 2.42934 |
| |
| 3.240756 0. |
| 2.0738726 3.3796282 |
| |
| 1.3811504 0.30889064 |
| 0. 0. |
| |
| 1.818252 0.9127424 |
| 1.1968298 0.4844057 |
| |
| 1.6940281 0. |
| 0. 0.10140109 |
| |
| 0. 0.2724784 |
| 0. 0. |
| |
| 1.5154634 0. |
| 2.1841986 2.747264 |
| |
| 0. 0.80568683 |
| 0.36282468 0. |
| |
| 1.35923 2.4812453 |
| 0. 0. |
| |
| 1.9370286 2.6119037 |
| 0. 0. ", |
| rows=N, cols=4*C_base*expected_Hout*expected_Wout) |
| |
| blocks_weights = list(weights_block1, weights_block2) |
| ema_means_vars = list(emas_block1, emas_block2) |
| C_in = 2 |
| block_type = "bottleneck" |
| [out, Hout, Wout, ema_means_vars_upd, cached_out, cached_means_vars] = resnet::reslayer_forward(X, Hin, Win, |
| block_type, blocks, strideh, stridew, C_in, C_base, blocks_weights, mode, ema_means_vars) |
| |
| test_util::check_all_close(out, expected_out, 0.0001) |
| } |
| |
| values_test_bottleneck_block_backward = function() { |
| /* |
| * Testing of values for forward pass of basic block against PyTorch. |
| */ |
| strideh = 2; stridew = 2 |
| C_in = 4; C_base = 3 |
| Hin = 4; Win = 4 |
| N = 2 |
| |
| # random inputs |
| X = matrix("0.6162815 0.4403988 0.19323945 0.20049834 |
| 0.05038476 0.21500802 0.34992313 0.5681402 |
| 0.6817395 0.46402705 0.796808 0.7984357 |
| 0.6264346 0.83064973 0.29703385 0.14875907 |
| |
| 0.38349533 0.9782912 0.34868628 0.94089437 |
| 0.38948953 0.4737754 0.2612831 0.08784401 |
| 0.34254092 0.70385265 0.98290646 0.86050016 |
| 0.73177606 0.75201535 0.6362604 0.53918016 |
| |
| 0.07957208 0.3877396 0.55845153 0.3644358 |
| 0.49124908 0.42955297 0.46360916 0.98667765 |
| 0.10313964 0.2232694 0.16390342 0.8962339 |
| 0.6431137 0.52672505 0.24231404 0.09009731 |
| |
| 0.17517161 0.72460824 0.10649377 0.5046083 |
| 0.75258034 0.206025 0.85093164 0.4466961 |
| 0.39183217 0.36186397 0.48932898 0.9048042 |
| 0.9153776 0.37588143 0.6029127 0.25502872 |
| |
| |
| 0.18585867 0.58759594 0.2678948 0.37187874 |
| 0.91700923 0.21069771 0.2234245 0.4910056 |
| 0.6757395 0.25290525 0.3931765 0.37943745 |
| 0.5380152 0.20728391 0.24641341 0.03234977 |
| |
| 0.5496058 0.07991606 0.21452975 0.5860226 |
| 0.8747525 0.70606184 0.7483169 0.77671623 |
| 0.12875003 0.07195783 0.8843897 0.4327739 |
| 0.6150542 0.1181671 0.4372782 0.44670075 |
| |
| 0.85027456 0.4657778 0.5655388 0.76092625 |
| 0.22075617 0.99045426 0.09102738 0.23654592 |
| 0.05107814 0.89564663 0.0411225 0.88786465 |
| 0.33387965 0.72221154 0.50403994 0.34668463 |
| |
| 0.6427465 0.49647993 0.5180299 0.08453029 |
| 0.1819312 0.39922637 0.62086684 0.07386869 |
| 0.6140939 0.8560102 0.65571284 0.07469827 |
| 0.9055746 0.86279285 0.00330186 0.32202858", rows=N, cols=C_in*Hin*Win) |
| W_conv1 = matrix(" 0.07869178 |
| |
| 0.31600833 |
| |
| 0.19698411 |
| |
| 0.2936622 |
| |
| |
| 0.22970712 |
| |
| 0.04483354 |
| |
| -0.30420136 |
| |
| -0.09719777 |
| |
| |
| 0.1794408 |
| |
| -0.20857847 |
| |
| -0.0734545 |
| |
| 0.19861615", rows=C_base, cols=C_in*1*1) |
| gamma_bn1 = matrix(1, rows=C_base, cols=1) |
| beta_bn1 = matrix(0, rows=C_base, cols=1) |
| W_conv2 = matrix(" 0.12317522 0.0246315 0.1567147 |
| -0.12465923 -0.03034407 0.12380241 |
| 0.16502933 0.01206452 0.03130221 |
| |
| -0.13969627 -0.15804827 0.07226227 |
| 0.16526262 -0.07034778 0.03869167 |
| -0.05009787 -0.05447315 -0.03771345 |
| |
| 0.04777487 -0.1882673 0.03379487 |
| -0.0138817 0.12698577 0.01845463 |
| -0.05744886 0.17056645 -0.14263153 |
| |
| |
| 0.13967635 -0.12141858 -0.16375405 |
| -0.00954916 0.08370475 -0.17045537 |
| -0.14925331 -0.05134124 -0.14130534 |
| |
| -0.1700635 0.01796004 0.09484367 |
| 0.1771652 -0.05563843 -0.03825609 |
| 0.04600334 0.17806517 0.15516032 |
| |
| 0.05288036 0.10429145 0.1735195 |
| -0.0042614 -0.04825242 -0.08205798 |
| -0.01969416 0.08754538 0.03742084 |
| |
| |
| 0.01199178 0.14878578 -0.0408629 |
| 0.01696147 0.19012083 -0.14739883 |
| 0.14986785 -0.13744283 0.00567606 |
| |
| 0.03487785 0.00860398 0.16434713 |
| -0.11653656 0.16149329 0.0224424 |
| 0.09091513 -0.01516792 -0.15144783 |
| |
| -0.00255089 -0.1327273 -0.11388822 |
| 0.11385046 -0.0044629 -0.18658024 |
| -0.00919266 0.13959612 -0.03503697", rows=C_base, cols=C_base*3*3) |
| gamma_bn2 = matrix(1, rows=C_base, cols=1) |
| beta_bn2 = matrix(0, rows=C_base, cols=1) |
| W_conv3 = matrix("-0.5393644 |
| |
| -0.19495046 |
| |
| -0.34656495 |
| |
| |
| -0.38642883 |
| |
| 0.04036182 |
| |
| -0.05183017 |
| |
| |
| -0.3516076 |
| |
| 0.37582028 |
| |
| -0.32641533 |
| |
| |
| -0.07366091 |
| |
| 0.49823952 |
| |
| -0.1703605 |
| |
| |
| 0.08072537 |
| |
| -0.05598992 |
| |
| 0.262644 |
| |
| |
| 0.549786 |
| |
| 0.43183482 |
| |
| 0.26671004 |
| |
| |
| 0.06465209 |
| |
| 0.06707138 |
| |
| 0.34622842 |
| |
| |
| 0.16586006 |
| |
| 0.45819652 |
| |
| -0.4871891 |
| |
| |
| 0.4211281 |
| |
| 0.49260843 |
| |
| -0.13952363 |
| |
| |
| -0.275389 |
| |
| 0.4940505 |
| |
| 0.14514804 |
| |
| |
| 0.51552415 |
| |
| 0.5144515 |
| |
| 0.52573395 |
| |
| |
| -0.17356852 |
| |
| 0.5181886 |
| |
| -0.50034744", rows=4*C_base, cols=C_base) |
| gamma_bn3 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn3 = matrix(0, rows=4*C_base, cols=1) |
| |
| W_conv4 = matrix("-2.24914551e-02 |
| |
| -3.20156872e-01 |
| |
| -2.57243097e-01 |
| |
| -2.83359289e-01 |
| |
| |
| 4.24459159e-01 |
| |
| 1.06867790e-01 |
| |
| 3.79524231e-02 |
| |
| 2.02206016e-01 |
| |
| |
| 2.60769248e-01 |
| |
| 1.12871289e-01 |
| |
| -4.35526133e-01 |
| |
| -3.94163132e-02 |
| |
| |
| -1.34141207e-01 |
| |
| 2.69388020e-01 |
| |
| 6.32327795e-02 |
| |
| -2.56273866e-01 |
| |
| |
| -1.53142989e-01 |
| |
| 2.02380061e-01 |
| |
| -6.39696121e-02 |
| |
| 2.35089958e-01 |
| |
| |
| 2.51878500e-01 |
| |
| -4.26316798e-01 |
| |
| -1.74328208e-01 |
| |
| 2.51074314e-01 |
| |
| |
| 4.09523904e-01 |
| |
| -2.02482343e-01 |
| |
| -1.13063693e-01 |
| |
| 4.72016454e-01 |
| |
| |
| 2.85914719e-01 |
| |
| 1.19355321e-01 |
| |
| 2.42567658e-01 |
| |
| -1.14123225e-02 |
| |
| |
| -3.01252782e-01 |
| |
| -3.56714487e-01 |
| |
| -1.66478992e-01 |
| |
| 3.00422072e-01 |
| |
| |
| 4.04710472e-01 |
| |
| 3.43372107e-01 |
| |
| 1.61813855e-01 |
| |
| 3.13257992e-01 |
| |
| |
| 3.18122804e-01 |
| |
| -7.92363882e-02 |
| |
| 2.41994858e-04 |
| |
| 1.33128226e-01 |
| |
| |
| 4.88885224e-01 |
| |
| 4.68040824e-01 |
| |
| 1.40627801e-01 |
| |
| -1.00752831e-01", rows=4*C_base, cols=C_in) |
| gamma_bn4 = matrix(1, rows=4*C_base, cols=1) |
| beta_bn4 = matrix(0, rows=4*C_base, cols=1) |
| |
| |
| weights = list(W_conv1, gamma_bn1, beta_bn1, W_conv2, gamma_bn2, beta_bn2, W_conv3, gamma_bn3, beta_bn3, W_conv4, |
| gamma_bn4, beta_bn4) |
| mode = "train" |
| |
| ema_mean_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn1 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_var_bn2 = matrix(0, rows=C_base, cols=1) |
| ema_mean_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn3 = matrix(0, rows=4*C_base, cols=1) |
| ema_mean_bn4 = matrix(0, rows=4*C_base, cols=1) |
| ema_var_bn4 = matrix(0, rows=4*C_base, cols=1) |
| ema_means_vars = list(ema_mean_bn1, ema_var_bn1, ema_mean_bn2, ema_var_bn2, ema_mean_bn3, ema_var_bn3, ema_mean_bn4, |
| ema_var_bn4) |
| |
| [out, Hout, Wout, ema_means_vars_up, cached_out, cached_means_vars] = resnet::bottleneck_block_forward(X, weights, |
| C_in, C_base, Hin, Win, strideh, stridew, mode, ema_means_vars) |
| |
| dOut = matrix("-0.00135712 0.01696673 |
| -0.01479281 -0.02083333 |
| |
| -0.02083333 -0.02083333 |
| -0.02083333 0.0113719 |
| |
| -0.02083333 -0.02083333 |
| -0.02083333 -0.00120236 |
| |
| -0.02083333 0.01352049 |
| -0.02083333 -0.01275505 |
| |
| -0.02083333 -0.02083333 |
| -0.02083333 -0.01103638 |
| |
| -0.01391222 -0.02083333 |
| 0.00343994 -0.02083333 |
| |
| -0.02083333 -0.02083333 |
| -0.01393827 -0.02083333 |
| |
| -0.02083333 -0.02083333 |
| -0.01800691 0.02834493 |
| |
| -0.02083333 -0.02083333 |
| -0.00615833 -0.02083333 |
| |
| -0.02083333 -0.02083333 |
| -0.02083333 0.00811108 |
| |
| -0.02083333 -0.02083333 |
| -0.00525803 -0.01419366 |
| |
| -0.02083333 -0.02083333 |
| -0.02083333 0.02624077 |
| |
| |
| -0.02083333 -0.02083333 |
| 0.0157299 -0.02083333 |
| |
| -0.00279292 -0.02083333 |
| -0.00570655 -0.00010767 |
| |
| -0.01608849 -0.02083333 |
| -0.01170836 -0.02083333 |
| |
| 0.02918951 -0.01127005 |
| -0.02083333 -0.00505277 |
| |
| -0.02083333 -0.02083333 |
| -0.02083333 0.07186925 |
| |
| -0.02083333 0.00095898 |
| -0.00224105 0.00826271 |
| |
| -0.02083333 -0.02083333 |
| 0.00410065 0.03495027 |
| |
| 0.02533473 -0.01610413 |
| -0.02083333 -0.02083333 |
| |
| 0.00404375 0.03972482 |
| -0.01891485 -0.02083333 |
| |
| 0.0060901 -0.02083333 |
| -0.02083333 0.03727701 |
| |
| -0.02083333 -0.02021644 |
| -0.00819749 0.02092233 |
| |
| 0.00930707 -0.02083333 |
| -0.02083333 -0.02083333", |
| rows=N, cols=4*C_base*Hout*Wout) |
| |
| [dX, gradients] = resnet::bottleneck_block_backward(dOut, cached_out, weights, C_in, C_base, |
| strideh, stridew, cached_means_vars) |
| |
| dW_conv1_exp = matrix("-0.41785866 |
| |
| 0.0931843 |
| |
| 0.34355345 |
| |
| -0.21892753 |
| |
| |
| 0.0086094 |
| |
| -0.04990812 |
| |
| -0.01500304 |
| |
| 0.0445025 |
| |
| |
| -0.30176648 |
| |
| -0.49171305 |
| |
| 0.26180965 |
| |
| -0.1460878 ", |
| rows=C_base, cols=C_in) |
| test_util::check_all_close(as.matrix(gradients[1]), dW_conv1_exp, 0.0001) |
| |
| dW_bn1_exp = matrix("-0.08518536 -0.03189257 0.11702712", rows=C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[2]), dW_bn1_exp, 0.0001) |
| |
| db_bn1_exp = matrix("0.00237629 0.00613181 0.09522828", rows=C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[3]), db_bn1_exp, 0.0001) |
| |
| dW_conv2_exp = matrix("-4.45271172e-02 -2.74588205e-02 7.13727996e-02 |
| 7.06704110e-02 -1.54908448e-01 -4.83400404e-01 |
| 2.65698969e-01 -1.25077665e-01 -1.10657856e-01 |
| |
| 0.00000000e+00 1.21304452e-01 -5.32024577e-02 |
| -2.83185914e-02 -5.62763751e-01 -1.45827040e-01 |
| -7.79439956e-02 1.06638774e-01 2.44921818e-01 |
| |
| 0.00000000e+00 -1.66312024e-01 -8.96646082e-02 |
| 4.92776275e-01 1.23488665e-01 -2.30740905e-02 |
| -6.43311292e-02 2.64928266e-02 1.40425444e-01 |
| |
| |
| 6.06015176e-02 4.26720977e-02 -2.87417118e-02 |
| 2.61508137e-01 3.17626178e-01 8.05854917e-01 |
| -1.89844340e-01 -1.56550437e-01 -1.41548123e-02 |
| |
| 0.00000000e+00 -2.49216743e-02 7.24086761e-02 |
| 2.08961636e-01 3.44124526e-01 -5.16285971e-02 |
| 2.13657036e-01 -8.95786211e-02 -3.35685104e-01 |
| |
| 0.00000000e+00 3.76305908e-01 2.45785102e-01 |
| -5.38561165e-01 -3.42951834e-01 1.81010082e-01 |
| 9.29689035e-02 1.27511114e-01 9.73739699e-02 |
| |
| |
| 8.97027832e-03 3.36519163e-03 2.44523492e-03 |
| 2.27232394e-03 1.78420525e-02 9.22706071e-03 |
| -4.80103632e-03 7.12600246e-04 3.36243305e-03 |
| |
| 0.00000000e+00 1.00413887e-02 1.07179824e-02 |
| 6.55718101e-03 1.13412477e-02 -1.87643571e-03 |
| 7.72099057e-03 2.41162861e-03 -9.22553055e-03 |
| |
| 0.00000000e+00 1.39963506e-02 8.88201315e-03 |
| -8.13074317e-03 -5.43177221e-03 1.47716720e-02 |
| 1.25582376e-02 2.33817846e-04 2.97963596e-03", |
| rows=C_base, cols=C_base*3*3) |
| test_util::check_all_close(as.matrix(gradients[4]), dW_conv2_exp, 0.0001) |
| |
| dW_bn2_exp = matrix("-0.06543966 0.04738945 0.018198", rows=C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[5]), dW_bn2_exp, 0.0001) |
| |
| db_bn2_exp = matrix("-0.06848848 0.11912203 0.00337613", rows=C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[6]), db_bn2_exp, 0.0001) |
| |
| dW_conv3_exp = matrix("-5.60766011e-02 |
| |
| 2.29696594e-02 |
| |
| 7.43360370e-02 |
| |
| |
| -1.86348334e-04 |
| |
| -1.88461086e-03 |
| |
| -2.28913268e-05 |
| |
| |
| 6.30268827e-04 |
| |
| 1.00867022e-02 |
| |
| 1.09486319e-02 |
| |
| |
| -2.18466222e-02 |
| |
| -4.36403081e-02 |
| |
| -1.18271664e-01 |
| |
| |
| -1.13629751e-01 |
| |
| 9.58261788e-02 |
| |
| 5.55922613e-02 |
| |
| |
| -2.95727067e-02 |
| |
| 2.25221030e-02 |
| |
| 2.45047137e-02 |
| |
| |
| -3.81892845e-02 |
| |
| 4.22266824e-03 |
| |
| 6.35173637e-03 |
| |
| |
| -8.05793107e-02 |
| |
| 3.31897177e-02 |
| |
| 3.77757102e-03 |
| |
| |
| -9.51854140e-02 |
| |
| 1.28410965e-01 |
| |
| 1.65754408e-01 |
| |
| |
| 1.40101882e-02 |
| |
| -1.29800066e-02 |
| |
| 7.07973465e-02 |
| |
| |
| -2.28845570e-02 |
| |
| -1.79107301e-03 |
| |
| 2.41976194e-02 |
| |
| |
| 7.98613299e-04 |
| |
| -1.13419816e-02 |
| |
| -1.20270848e-02", rows=4*C_base, cols=C_base) |
| test_util::check_all_close(as.matrix(gradients[7]), dW_conv3_exp, 0.0001) |
| |
| dW_bn3_exp = matrix("0.04131351 -0.00676661 -0.02855339 0.05521806 0.18712975 0.02106027 |
| 0.09423574 0.01858789 0.09263451 0.07307401 0.05661228 0.01685416", rows=4*C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[8]), dW_bn3_exp, 0.0001) |
| |
| db_bn3_exp = matrix("0.0165467 0.00276476 -0.02899921 0.01363214 0.06083286 -0.00349163 |
| 0.02511264 0.01956862 0.01869539 0.0514782 -0.02694329 0.03554784", rows=4*C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[9]), db_bn3_exp, 0.0001) |
| |
| dW_conv4_exp = matrix("-0.04296157 |
| |
| -0.0239915 |
| |
| 0.03860337 |
| |
| -0.00455155 |
| |
| |
| -0.00125965 |
| |
| 0.04469311 |
| |
| 0.01571925 |
| |
| -0.0238542 |
| |
| |
| -0.01008898 |
| |
| 0.00347374 |
| |
| -0.00268455 |
| |
| -0.02727252 |
| |
| |
| -0.13418952 |
| |
| -0.11058047 |
| |
| 0.19957161 |
| |
| 0.00311983 |
| |
| |
| -0.07345656 |
| |
| -0.06987098 |
| |
| -0.21337292 |
| |
| -0.04499394 |
| |
| |
| -0.008177 |
| |
| 0.01877422 |
| |
| -0.00474093 |
| |
| 0.03676792 |
| |
| |
| -0.03318542 |
| |
| 0.09919735 |
| |
| -0.05045987 |
| |
| 0.05925765 |
| |
| |
| -0.0827354 |
| |
| 0.09468257 |
| |
| 0.05471547 |
| |
| 0.05080906 |
| |
| |
| -0.10927054 |
| |
| 0.01890075 |
| |
| 0.1514071 |
| |
| -0.0031732 |
| |
| |
| -0.05194847 |
| |
| 0.05187422 |
| |
| -0.02254089 |
| |
| 0.02196812 |
| |
| |
| 0.01654913 |
| |
| 0.13097204 |
| |
| -0.1566386 |
| |
| 0.03831299 |
| |
| |
| -0.01303891 |
| |
| 0.00028254 |
| |
| 0.05167245 |
| |
| 0.00996144", rows=4*C_base, cols=C_in) |
| test_util::check_all_close(as.matrix(gradients[10]), dW_conv4_exp, 0.0001) |
| |
| dW_bn4_exp = matrix(" 0.01151968 0.01767683 0.01862798 0.02321731 0.12747863 -0.01113123 |
| -0.00035772 0.09836673 0.02158613 0.05004199 -0.02870339 0.05590326", |
| rows=4*C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[11]), dW_bn4_exp, 0.0001) |
| |
| db_bn4_exp = matrix("0.0165467 0.00276476 -0.02899921 0.01363214 0.06083286 -0.00349163 |
| 0.02511264 0.01956862 0.01869539 0.0514782 -0.02694329 0.03554784", |
| rows=4*C_base, cols=1) |
| test_util::check_all_close(as.matrix(gradients[12]), db_bn4_exp, 0.0001) |
| } |
| |
| |
| /* |
| * **** Bottleneck Block Value Testing **** |
| * In these test cases, we compare the forward pass |
| * computation of a basic residual block against the |
| * PyTorch implementation. We calculate the PyTorch |
| * values with the NN module |
| * torchvision.models.resnet.Bottleneck and then |
| * hard-code the randomly initialized weights and |
| * biases and the expected output computed by PyTorch |
| * into this file. |
| */ |
| |
| /* |
| * Test case 1: |
| * A forward pass of the bottleneck block with same |
| * input and output dimensions, i.e. stride 1 but |
| * with the default block expansion, i.e. different |
| * number of input and output channels thus requiring |
| * downsampling, in train mode. |
| */ |
| values_test_bottleneck_forward_1() |
| |
| /* |
| * Test case 2: |
| * A forward pass of the bottleneck block with different |
| * input and output dimensions with stride of 2 and |
| * with the default block expansion, i.e. downsampling, |
| * in train mode. |
| */ |
| values_test_bottleneck_forward_2() |
| |
| /* |
| * *** Residual Layer Value Testing *** |
| * A residual layer is a sequence of residual blocks |
| * which all have the same number of base channels. In |
| * residual networks, there are 4 different residual layer. |
| * With this test, we test the correct computation of the |
| * shape and values of the output by comparing it to PyTorches |
| * residual layers. We modified the PyTorch implementation to |
| * extract the residual layer. |
| */ |
| |
| /* |
| * Test case 1: |
| * A residual layer forward pass with 2 bottleneck residual |
| * blocks with a stride of 2 and the default block expansion |
| * of 4. |
| */ |
| values_test_residual_layer_forward() |
| |
| /* |
| * *** Residual Layer Backward Pass Testing *** |
| */ |
| |
| values_test_bottleneck_block_backward() |
| |
