| /*********************************************************************/ |
| /* Copyright 2009, 2010 The University of Texas at Austin. */ |
| /* All rights reserved. */ |
| /* */ |
| /* Redistribution and use in source and binary forms, with or */ |
| /* without modification, are permitted provided that the following */ |
| /* conditions are met: */ |
| /* */ |
| /* 1. Redistributions of source code must retain the above */ |
| /* copyright notice, this list of conditions and the following */ |
| /* disclaimer. */ |
| /* */ |
| /* 2. Redistributions in binary form must reproduce the above */ |
| /* copyright notice, this list of conditions and the following */ |
| /* disclaimer in the documentation and/or other materials */ |
| /* provided with the distribution. */ |
| /* */ |
| /* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ |
| /* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ |
| /* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ |
| /* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ |
| /* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ |
| /* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ |
| /* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ |
| /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ |
| /* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ |
| /* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ |
| /* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ |
| /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ |
| /* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ |
| /* POSSIBILITY OF SUCH DAMAGE. */ |
| /* */ |
| /* The views and conclusions contained in the software and */ |
| /* documentation are those of the authors and should not be */ |
| /* interpreted as representing official policies, either expressed */ |
| /* or implied, of The University of Texas at Austin. */ |
| /*********************************************************************/ |
| |
| #define ASSEMBLER |
| #include "common.h" |
| |
| #ifndef __64BIT__ |
| #define LOAD lwz |
| #else |
| #define LOAD ld |
| #endif |
| |
| #ifdef __64BIT__ |
| #define STACKSIZE 320 |
| #define ALPHA 296(SP) |
| #define FZERO 304(SP) |
| #else |
| #define STACKSIZE 240 |
| #define ALPHA 224(SP) |
| #define FZERO 232(SP) |
| #endif |
| |
| #define M r3 |
| #define N r4 |
| #define K r5 |
| |
| #if defined(linux) || defined(__FreeBSD__) |
| #ifndef __64BIT__ |
| #define A r6 |
| #define B r7 |
| #define C r8 |
| #define LDC r9 |
| #define OFFSET r10 |
| #else |
| #define A r7 |
| #define B r8 |
| #define C r9 |
| #define LDC r10 |
| #define OFFSET r6 |
| #endif |
| #endif |
| |
| #if defined(_AIX) || defined(__APPLE__) |
| #if !defined(__64BIT__) && defined(DOUBLE) |
| #define A r8 |
| #define B r9 |
| #define C r10 |
| #define LDC r7 |
| #define OFFSET r6 |
| #else |
| #define A r7 |
| #define B r8 |
| #define C r9 |
| #define LDC r10 |
| #define OFFSET r6 |
| #endif |
| #endif |
| |
| #define TEMP r18 |
| #define KK r19 |
| #define BB r20 |
| #define I r21 |
| #define J r22 |
| #define AO r23 |
| #define BO r24 |
| #define CO1 r25 |
| #define CO2 r26 |
| #define CO3 r27 |
| #define CO4 r28 |
| |
| #define PREA r29 |
| #define PREB r30 |
| #define PREC r31 |
| |
| #ifndef NEEDPARAM |
| |
| PROLOGUE |
| PROFCODE |
| |
| addi SP, SP, -STACKSIZE |
| li r0, 0 |
| |
| stfd f14, 0(SP) |
| stfd f15, 8(SP) |
| stfd f16, 16(SP) |
| stfd f17, 24(SP) |
| |
| stfd f18, 32(SP) |
| stfd f19, 40(SP) |
| stfd f20, 48(SP) |
| stfd f21, 56(SP) |
| |
| stfd f22, 64(SP) |
| stfd f23, 72(SP) |
| stfd f24, 80(SP) |
| stfd f25, 88(SP) |
| |
| stfd f26, 96(SP) |
| stfd f27, 104(SP) |
| stfd f28, 112(SP) |
| stfd f29, 120(SP) |
| |
| stfd f30, 128(SP) |
| stfd f31, 136(SP) |
| |
| #ifdef __64BIT__ |
| std r31, 144(SP) |
| std r30, 152(SP) |
| std r29, 160(SP) |
| std r28, 168(SP) |
| std r27, 176(SP) |
| std r26, 184(SP) |
| std r25, 192(SP) |
| std r24, 200(SP) |
| std r23, 208(SP) |
| std r22, 216(SP) |
| std r21, 224(SP) |
| std r20, 232(SP) |
| #if defined(TRMMKERNEL) |
| std r19, 240(SP) |
| std r18, 248(SP) |
| #endif |
| #else |
| stw r31, 144(SP) |
| stw r30, 148(SP) |
| stw r29, 152(SP) |
| stw r28, 156(SP) |
| stw r27, 160(SP) |
| stw r26, 164(SP) |
| stw r25, 168(SP) |
| stw r24, 172(SP) |
| stw r23, 176(SP) |
| stw r22, 180(SP) |
| stw r21, 184(SP) |
| stw r20, 188(SP) |
| #if defined(TRMMKERNEL) |
| stw r19, 192(SP) |
| stw r18, 196(SP) |
| #endif |
| #endif |
| |
| stfd f1, ALPHA |
| stw r0, FZERO |
| |
| #if defined(_AIX) || defined(__APPLE__) |
| #if !defined(__64BIT__) && defined(DOUBLE) |
| lwz LDC, FRAMESLOT(0) + STACKSIZE(SP) |
| #endif |
| #endif |
| |
| slwi LDC, LDC, BASE_SHIFT |
| |
| #if defined(TRMMKERNEL) |
| #if (defined(linux) || defined(__FreeBSD__)) && defined(__64BIT__) |
| ld OFFSET, FRAMESLOT(0) + STACKSIZE(SP) |
| #endif |
| |
| #if defined(_AIX) || defined(__APPLE__) |
| #ifdef __64BIT__ |
| ld OFFSET, FRAMESLOT(0) + STACKSIZE(SP) |
| #else |
| #ifdef DOUBLE |
| lwz OFFSET, FRAMESLOT(1) + STACKSIZE(SP) |
| #else |
| lwz OFFSET, FRAMESLOT(0) + STACKSIZE(SP) |
| #endif |
| #endif |
| #endif |
| #endif |
| |
| #if defined(TRMMKERNEL) && !defined(LEFT) |
| neg KK, OFFSET |
| #endif |
| |
| cmpwi cr0, M, 0 |
| ble LL(999) |
| cmpwi cr0, N, 0 |
| ble LL(999) |
| cmpwi cr0, K, 0 |
| ble LL(999) |
| |
| #ifndef PREFETCHTEST |
| /* Normal prefetch */ |
| #ifdef PPC970 |
| li PREC, 4 * SIZE |
| #endif |
| #ifdef POWER4 |
| li PREC, 4 * SIZE /* is 12 best? */ |
| #endif |
| #ifdef POWER5 |
| li PREC, 3 * SIZE |
| #endif |
| |
| #else |
| |
| #if defined(linux) || defined(__FreeBSD__) |
| #ifndef __64BIT__ |
| mr PREA, r10 |
| lwz PREB, FRAMESLOT(0) + STACKSIZE(SP) |
| lwz PREC, FRAMESLOT(1) + STACKSIZE(SP) |
| #else |
| ld PREA, FRAMESLOT(0) + STACKSIZE(SP) |
| ld PREB, FRAMESLOT(1) + STACKSIZE(SP) |
| ld PREC, FRAMESLOT(2) + STACKSIZE(SP) |
| #endif |
| #endif |
| |
| #if defined(_AIX) || defined(__APPLE__) |
| #ifdef __64BIT__ |
| ld PREA, FRAMESLOT(0) + STACKSIZE(SP) |
| ld PREB, FRAMESLOT(1) + STACKSIZE(SP) |
| ld PREC, FRAMESLOT(2) + STACKSIZE(SP) |
| #else |
| #ifdef DOUBLE |
| lwz PREA, FRAMESLOT(1) + STACKSIZE(SP) |
| lwz PREB, FRAMESLOT(2) + STACKSIZE(SP) |
| lwz PREC, FRAMESLOT(3) + STACKSIZE(SP) |
| #else |
| lwz PREA, FRAMESLOT(0) + STACKSIZE(SP) |
| lwz PREB, FRAMESLOT(1) + STACKSIZE(SP) |
| lwz PREC, FRAMESLOT(2) + STACKSIZE(SP) |
| #endif |
| #endif |
| #endif |
| |
| #endif |
| |
| #ifndef PREFETCHTEST |
| #ifdef PPC970 |
| #ifdef ALLOC_HUGETLB |
| li PREA, (16 * 1 * SIZE) |
| li PREB, (16 * 5 * SIZE) |
| #else |
| li PREA, (16 * 19 * SIZE) |
| li PREB, (16 * 8 * SIZE) |
| #endif |
| #endif |
| #ifdef POWER4 |
| #ifdef ALLOC_HUGETLB |
| li PREA, (16 * 1 * SIZE) |
| li PREB, (16 * 1 * SIZE) |
| #else |
| li PREA, (16 * 2 * SIZE) |
| li PREB, (16 * 2 * SIZE) |
| #endif |
| #endif |
| #ifdef POWER5 |
| #ifdef ALLOC_HUGETLB |
| li PREA, (16 * 7 * SIZE) |
| li PREB, (16 * 7 * SIZE) |
| #else |
| li PREA, (16 * 12 * SIZE) |
| li PREB, (16 * 6 * SIZE) |
| #endif |
| #endif |
| #endif |
| |
| srawi. J, N, 2 |
| ble LL(40) |
| .align 4 |
| |
| LL(10): |
| mr CO1, C |
| add CO2, C, LDC |
| add CO3, CO2, LDC |
| add CO4, CO3, LDC |
| |
| #if defined(TRMMKERNEL) && defined(LEFT) |
| mr KK, OFFSET |
| #endif |
| |
| slwi BB, K, BASE_SHIFT + 2 |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| fmr f4, f0 |
| fmr f5, f0 |
| fmr f6, f0 |
| fmr f7, f0 |
| fmr f8, f0 |
| fmr f9, f0 |
| fmr f10, f0 |
| fmr f11, f0 |
| fmr f12, f0 |
| fmr f13, f0 |
| fmr f14, f0 |
| fmr f15, f0 |
| |
| srawi. I, M, 2 |
| mr AO, A |
| add C, CO4, LDC |
| ble LL(20) |
| .align 4 |
| |
| LL(11): |
| #if defined(TRMMKERNEL) |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| #ifdef POWER5 |
| LFD f28, 4 * SIZE(B) |
| LFD f29, 5 * SIZE(B) |
| LFD f30, 6 * SIZE(B) |
| LFD f31, 7 * SIZE(B) |
| #endif |
| mr BO, B |
| #else |
| slwi r0, KK, 2 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, r0 |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| |
| #ifdef POWER5 |
| LFD f28, 4 * SIZE(BO) |
| LFD f29, 5 * SIZE(BO) |
| LFD f30, 6 * SIZE(BO) |
| LFD f31, 7 * SIZE(BO) |
| #endif |
| #endif |
| |
| DCBTST(CO1, PREC) |
| DCBTST(CO2, PREC) |
| DCBTST(CO3, PREC) |
| DCBTST(CO4, PREC) |
| |
| dcbt B, BB |
| addi BB, BB, 16 * SIZE |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 4 |
| #else |
| addi TEMP, KK, 4 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| ble LL(15) |
| |
| #else |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| #ifdef POWER5 |
| LFD f28, 4 * SIZE(B) |
| LFD f29, 5 * SIZE(B) |
| LFD f30, 6 * SIZE(B) |
| LFD f31, 7 * SIZE(B) |
| #endif |
| |
| DCBTST(CO1, PREC) |
| DCBTST(CO2, PREC) |
| DCBTST(CO3, PREC) |
| DCBTST(CO4, PREC) |
| |
| dcbt B, BB |
| addi BB, BB, 16 * SIZE |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| ble LL(15) |
| #endif |
| .align 4 |
| |
| LL(12): |
| FMADD f0, f16, f20, f0 |
| FMADD f5, f17, f21, f5 |
| FMADD f10, f18, f22, f10 |
| FMADD f15, f19, f23, f15 |
| |
| #if defined(ALLOC_HUGETLB) && !defined(POWER5) |
| LFD f28, 4 * SIZE(BO) |
| LFD f29, 5 * SIZE(BO) |
| LFD f30, 6 * SIZE(BO) |
| LFD f31, 7 * SIZE(BO) |
| #endif |
| |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| FMADD f4, f16, f21, f4 |
| |
| #if !defined(ALLOC_HUGETLB) && !defined(POWER5) |
| LFD f28, 4 * SIZE(BO) |
| LFD f29, 5 * SIZE(BO) |
| LFD f30, 6 * SIZE(BO) |
| LFD f31, 7 * SIZE(BO) |
| #endif |
| |
| LFD f24, 4 * SIZE(AO) |
| LFD f25, 5 * SIZE(AO) |
| LFD f26, 6 * SIZE(AO) |
| LFD f27, 7 * SIZE(AO) |
| |
| FMADD f6, f18, f21, f6 |
| FMADD f7, f19, f21, f7 |
| FMADD f8, f16, f22, f8 |
| FMADD f9, f17, f22, f9 |
| |
| FMADD f11, f19, f22, f11 |
| FMADD f12, f16, f23, f12 |
| FMADD f13, f17, f23, f13 |
| FMADD f14, f18, f23, f14 |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| FMADD f0, f24, f28, f0 |
| FMADD f5, f25, f29, f5 |
| FMADD f10, f26, f30, f10 |
| FMADD f15, f27, f31, f15 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| FMADD f1, f25, f28, f1 |
| FMADD f2, f26, f28, f2 |
| FMADD f3, f27, f28, f3 |
| FMADD f4, f24, f29, f4 |
| |
| FMADD f6, f26, f29, f6 |
| FMADD f7, f27, f29, f7 |
| FMADD f8, f24, f30, f8 |
| FMADD f9, f25, f30, f9 |
| |
| FMADD f11, f27, f30, f11 |
| FMADD f12, f24, f31, f12 |
| FMADD f13, f25, f31, f13 |
| FMADD f14, f26, f31, f14 |
| |
| LFD f28, 12 * SIZE(BO) |
| LFD f29, 13 * SIZE(BO) |
| LFD f30, 14 * SIZE(BO) |
| LFD f31, 15 * SIZE(BO) |
| |
| FMADD f0, f16, f20, f0 |
| FMADD f5, f17, f21, f5 |
| FMADD f10, f18, f22, f10 |
| FMADD f15, f19, f23, f15 |
| |
| LFD f24, 12 * SIZE(AO) |
| LFD f25, 13 * SIZE(AO) |
| LFD f26, 14 * SIZE(AO) |
| LFD f27, 15 * SIZE(AO) |
| |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| FMADD f4, f16, f21, f4 |
| |
| FMADD f6, f18, f21, f6 |
| FMADD f7, f19, f21, f7 |
| FMADD f8, f16, f22, f8 |
| FMADD f9, f17, f22, f9 |
| |
| FMADD f11, f19, f22, f11 |
| FMADD f12, f16, f23, f12 |
| FMADD f13, f17, f23, f13 |
| FMADD f14, f18, f23, f14 |
| |
| #ifndef POWER5 |
| LFD f16, 16 * SIZE(AO) |
| LFD f17, 17 * SIZE(AO) |
| LFD f18, 18 * SIZE(AO) |
| LFD f19, 19 * SIZE(AO) |
| #else |
| LFD f20, 16 * SIZE(BO) |
| LFD f21, 17 * SIZE(BO) |
| LFD f22, 18 * SIZE(BO) |
| LFD f23, 19 * SIZE(BO) |
| #endif |
| |
| FMADD f0, f24, f28, f0 |
| FMADD f5, f25, f29, f5 |
| FMADD f10, f26, f30, f10 |
| FMADD f15, f27, f31, f15 |
| |
| #ifndef POWER5 |
| LFD f20, 16 * SIZE(BO) |
| LFD f21, 17 * SIZE(BO) |
| LFD f22, 18 * SIZE(BO) |
| LFD f23, 19 * SIZE(BO) |
| #else |
| LFD f16, 16 * SIZE(AO) |
| LFD f17, 17 * SIZE(AO) |
| LFD f18, 18 * SIZE(AO) |
| LFD f19, 19 * SIZE(AO) |
| #endif |
| |
| FMADD f1, f25, f28, f1 |
| FMADD f2, f26, f28, f2 |
| FMADD f3, f27, f28, f3 |
| FMADD f4, f24, f29, f4 |
| |
| FMADD f6, f26, f29, f6 |
| FMADD f7, f27, f29, f7 |
| FMADD f8, f24, f30, f8 |
| FMADD f9, f25, f30, f9 |
| |
| FMADD f11, f27, f30, f11 |
| FMADD f12, f24, f31, f12 |
| FMADD f13, f25, f31, f13 |
| FMADD f14, f26, f31, f14 |
| |
| #if (L2_SIZE == 1024976) && defined (ALLOC_HUGETLB) |
| nop |
| nop |
| nop |
| nop |
| #endif |
| |
| #ifdef POWER5 |
| LFD f28, 20 * SIZE(BO) |
| LFD f29, 21 * SIZE(BO) |
| LFD f30, 22 * SIZE(BO) |
| LFD f31, 23 * SIZE(BO) |
| #endif |
| |
| addi AO, AO, 16 * SIZE |
| addi BO, BO, 16 * SIZE |
| |
| #ifdef PPC970 |
| #ifndef ALLOC_HUGETLB |
| DCBT(AO, PREA) |
| #endif |
| DCBT(BO, PREB) |
| #endif |
| |
| #ifdef POWER4 |
| #ifndef ALLOC_HUGETLB |
| DCBT(AO, PREA) |
| #endif |
| DCBT(BO, PREB) |
| #endif |
| |
| #ifdef POWER5 |
| #ifndef ALLOC_HUGETLB |
| DCBT(BO, PREB) |
| DCBT(AO, PREA) |
| #endif |
| #endif |
| bdnz LL(12) |
| .align 4 |
| |
| LL(15): |
| lfd f30, ALPHA |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 4 |
| #else |
| addi TEMP, KK, 4 |
| #endif |
| |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| #else |
| |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(18) |
| .align 4 |
| |
| LL(16): |
| FMADD f0, f16, f20, f0 |
| FMADD f5, f17, f21, f5 |
| FMADD f10, f18, f22, f10 |
| FMADD f15, f19, f23, f15 |
| |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| FMADD f4, f16, f21, f4 |
| |
| FMADD f6, f18, f21, f6 |
| FMADD f7, f19, f21, f7 |
| FMADD f8, f16, f22, f8 |
| FMADD f9, f17, f22, f9 |
| |
| FMADD f11, f19, f22, f11 |
| FMADD f12, f16, f23, f12 |
| FMADD f13, f17, f23, f13 |
| FMADD f14, f18, f23, f14 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| addi BO, BO, 4 * SIZE |
| addi AO, AO, 4 * SIZE |
| bdnz LL(16) |
| .align 4 |
| |
| LL(18): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f17, 1 * SIZE(CO1) |
| LFD f18, 2 * SIZE(CO1) |
| LFD f19, 3 * SIZE(CO1) |
| |
| LFD f20, 0 * SIZE(CO2) |
| LFD f21, 1 * SIZE(CO2) |
| LFD f22, 2 * SIZE(CO2) |
| LFD f23, 3 * SIZE(CO2) |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f17 |
| FMADD f2, f2, f30, f18 |
| FMADD f3, f3, f30, f19 |
| |
| FMADD f4, f4, f30, f20 |
| FMADD f5, f5, f30, f21 |
| FMADD f6, f6, f30, f22 |
| FMADD f7, f7, f30, f23 |
| |
| LFD f16, 0 * SIZE(CO3) |
| LFD f17, 1 * SIZE(CO3) |
| LFD f18, 2 * SIZE(CO3) |
| LFD f19, 3 * SIZE(CO3) |
| |
| LFD f20, 0 * SIZE(CO4) |
| LFD f21, 1 * SIZE(CO4) |
| LFD f22, 2 * SIZE(CO4) |
| LFD f23, 3 * SIZE(CO4) |
| |
| FMADD f8, f8, f30, f16 |
| FMADD f9, f9, f30, f17 |
| FMADD f10, f10, f30, f18 |
| FMADD f11, f11, f30, f19 |
| |
| FMADD f12, f12, f30, f20 |
| FMADD f13, f13, f30, f21 |
| FMADD f14, f14, f30, f22 |
| FMADD f15, f15, f30, f23 |
| |
| #else |
| |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| FMUL f2, f2, f30 |
| FMUL f3, f3, f30 |
| |
| FMUL f4, f4, f30 |
| FMUL f5, f5, f30 |
| FMUL f6, f6, f30 |
| FMUL f7, f7, f30 |
| |
| FMUL f8, f8, f30 |
| FMUL f9, f9, f30 |
| FMUL f10, f10, f30 |
| FMUL f11, f11, f30 |
| |
| FMUL f12, f12, f30 |
| FMUL f13, f13, f30 |
| FMUL f14, f14, f30 |
| FMUL f15, f15, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 1 * SIZE(CO1) |
| STFD f2, 2 * SIZE(CO1) |
| STFD f3, 3 * SIZE(CO1) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| STFD f4, 0 * SIZE(CO2) |
| STFD f5, 1 * SIZE(CO2) |
| STFD f6, 2 * SIZE(CO2) |
| STFD f7, 3 * SIZE(CO2) |
| |
| fmr f4, f0 |
| fmr f5, f0 |
| fmr f6, f0 |
| fmr f7, f0 |
| |
| STFD f8, 0 * SIZE(CO3) |
| STFD f9, 1 * SIZE(CO3) |
| STFD f10, 2 * SIZE(CO3) |
| STFD f11, 3 * SIZE(CO3) |
| |
| fmr f8, f0 |
| fmr f9, f0 |
| fmr f10, f0 |
| fmr f11, f0 |
| |
| STFD f12, 0 * SIZE(CO4) |
| STFD f13, 1 * SIZE(CO4) |
| STFD f14, 2 * SIZE(CO4) |
| STFD f15, 3 * SIZE(CO4) |
| |
| fmr f12, f0 |
| fmr f13, f0 |
| fmr f14, f0 |
| fmr f15, f0 |
| |
| addi CO1, CO1, 4 * SIZE |
| addi CO2, CO2, 4 * SIZE |
| addi CO3, CO3, 4 * SIZE |
| addi CO4, CO4, 4 * SIZE |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -4 |
| #else |
| addi TEMP, TEMP, -4 |
| #endif |
| slwi TEMP, TEMP, 2 + BASE_SHIFT |
| add AO, AO, TEMP |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 4 |
| #endif |
| #endif |
| |
| addic. I, I, -1 |
| bgt+ LL(11) |
| .align 4 |
| |
| LL(20): |
| andi. I, M, 2 |
| ble LL(30) |
| |
| #if defined(TRMMKERNEL) |
| #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 1 + BASE_SHIFT |
| slwi TEMP, KK, 2 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| |
| LFD f24, 4 * SIZE(BO) |
| LFD f25, 5 * SIZE(BO) |
| LFD f26, 6 * SIZE(BO) |
| LFD f27, 7 * SIZE(BO) |
| #endif |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 2 |
| #else |
| addi TEMP, KK, 4 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(25) |
| .align 5 |
| |
| LL(22): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f4, f16, f21, f4 |
| FMADD f5, f17, f21, f5 |
| |
| FMADD f8, f16, f22, f8 |
| FMADD f9, f17, f22, f9 |
| FMADD f12, f16, f23, f12 |
| FMADD f13, f17, f23, f13 |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| FMADD f2, f18, f24, f2 |
| FMADD f3, f19, f24, f3 |
| FMADD f6, f18, f25, f6 |
| FMADD f7, f19, f25, f7 |
| |
| FMADD f10, f18, f26, f10 |
| FMADD f11, f19, f26, f11 |
| FMADD f14, f18, f27, f14 |
| FMADD f15, f19, f27, f15 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f4, f16, f21, f4 |
| FMADD f5, f17, f21, f5 |
| |
| LFD f24, 12 * SIZE(BO) |
| LFD f25, 13 * SIZE(BO) |
| LFD f26, 14 * SIZE(BO) |
| LFD f27, 15 * SIZE(BO) |
| |
| FMADD f8, f16, f22, f8 |
| FMADD f9, f17, f22, f9 |
| FMADD f12, f16, f23, f12 |
| FMADD f13, f17, f23, f13 |
| |
| LFD f20, 16 * SIZE(BO) |
| LFD f21, 17 * SIZE(BO) |
| LFD f22, 18 * SIZE(BO) |
| LFD f23, 19 * SIZE(BO) |
| |
| FMADD f2, f18, f24, f2 |
| FMADD f3, f19, f24, f3 |
| FMADD f6, f18, f25, f6 |
| FMADD f7, f19, f25, f7 |
| |
| FMADD f10, f18, f26, f10 |
| FMADD f11, f19, f26, f11 |
| FMADD f14, f18, f27, f14 |
| FMADD f15, f19, f27, f15 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| LFD f24, 20 * SIZE(BO) |
| LFD f25, 21 * SIZE(BO) |
| LFD f26, 22 * SIZE(BO) |
| LFD f27, 23 * SIZE(BO) |
| |
| addi AO, AO, 8 * SIZE |
| addi BO, BO, 16 * SIZE |
| DCBT(BO, PREB) |
| bdnz LL(22) |
| |
| fadd f0, f2, f0 |
| fadd f1, f3, f1 |
| fadd f4, f6, f4 |
| fadd f5, f7, f5 |
| fadd f8, f10, f8 |
| fadd f9, f11, f9 |
| fadd f12, f14, f12 |
| fadd f13, f15, f13 |
| .align 4 |
| |
| LL(25): |
| lfd f30, ALPHA |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 2 |
| #else |
| addi TEMP, KK, 4 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(28) |
| .align 4 |
| |
| LL(26): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f4, f16, f21, f4 |
| FMADD f5, f17, f21, f5 |
| |
| FMADD f8, f16, f22, f8 |
| FMADD f9, f17, f22, f9 |
| FMADD f12, f16, f23, f12 |
| FMADD f13, f17, f23, f13 |
| |
| LFD f16, 2 * SIZE(AO) |
| LFD f17, 3 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| addi BO, BO, 4 * SIZE |
| addi AO, AO, 2 * SIZE |
| bdnz LL(26) |
| .align 4 |
| |
| LL(28): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f17, 1 * SIZE(CO1) |
| LFD f18, 0 * SIZE(CO2) |
| LFD f19, 1 * SIZE(CO2) |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f17 |
| FMADD f4, f4, f30, f18 |
| FMADD f5, f5, f30, f19 |
| |
| LFD f20, 0 * SIZE(CO3) |
| LFD f21, 1 * SIZE(CO3) |
| LFD f22, 0 * SIZE(CO4) |
| LFD f23, 1 * SIZE(CO4) |
| |
| FMADD f8, f8, f30, f20 |
| FMADD f9, f9, f30, f21 |
| FMADD f12, f12, f30, f22 |
| FMADD f13, f13, f30, f23 |
| #else |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| FMUL f4, f4, f30 |
| FMUL f5, f5, f30 |
| |
| FMUL f8, f8, f30 |
| FMUL f9, f9, f30 |
| FMUL f12, f12, f30 |
| FMUL f13, f13, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 1 * SIZE(CO1) |
| STFD f4, 0 * SIZE(CO2) |
| STFD f5, 1 * SIZE(CO2) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| STFD f8, 0 * SIZE(CO3) |
| STFD f9, 1 * SIZE(CO3) |
| STFD f12, 0 * SIZE(CO4) |
| STFD f13, 1 * SIZE(CO4) |
| |
| fmr f4, f0 |
| fmr f5, f0 |
| fmr f6, f0 |
| fmr f7, f0 |
| |
| fmr f8, f0 |
| fmr f9, f0 |
| fmr f10, f0 |
| fmr f11, f0 |
| |
| fmr f12, f0 |
| fmr f13, f0 |
| fmr f14, f0 |
| fmr f15, f0 |
| |
| addi CO1, CO1, 2 * SIZE |
| addi CO2, CO2, 2 * SIZE |
| addi CO3, CO3, 2 * SIZE |
| addi CO4, CO4, 2 * SIZE |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -2 |
| #else |
| addi TEMP, TEMP, -4 |
| #endif |
| slwi r0, TEMP, 1 + BASE_SHIFT |
| slwi TEMP, TEMP, 2 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 2 |
| #endif |
| #endif |
| .align 4 |
| |
| LL(30): |
| andi. I, M, 1 |
| ble LL(39) |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 0 + BASE_SHIFT |
| slwi TEMP, KK, 2 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| |
| LFD f24, 4 * SIZE(BO) |
| LFD f25, 5 * SIZE(BO) |
| LFD f26, 6 * SIZE(BO) |
| LFD f27, 7 * SIZE(BO) |
| #endif |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 1 |
| #else |
| addi TEMP, KK, 4 |
| #endif |
| |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(35) |
| .align 5 |
| |
| LL(32): |
| FMADD f0, f16, f20, f0 |
| FMADD f4, f16, f21, f4 |
| FMADD f8, f16, f22, f8 |
| FMADD f12, f16, f23, f12 |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| FMADD f1, f17, f24, f1 |
| FMADD f5, f17, f25, f5 |
| FMADD f9, f17, f26, f9 |
| FMADD f13, f17, f27, f13 |
| |
| LFD f24, 12 * SIZE(BO) |
| LFD f25, 13 * SIZE(BO) |
| LFD f26, 14 * SIZE(BO) |
| LFD f27, 15 * SIZE(BO) |
| |
| FMADD f0, f18, f20, f0 |
| FMADD f4, f18, f21, f4 |
| FMADD f8, f18, f22, f8 |
| FMADD f12, f18, f23, f12 |
| |
| LFD f20, 16 * SIZE(BO) |
| LFD f21, 17 * SIZE(BO) |
| LFD f22, 18 * SIZE(BO) |
| LFD f23, 19 * SIZE(BO) |
| |
| FMADD f1, f19, f24, f1 |
| FMADD f5, f19, f25, f5 |
| FMADD f9, f19, f26, f9 |
| FMADD f13, f19, f27, f13 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f24, 20 * SIZE(BO) |
| LFD f25, 21 * SIZE(BO) |
| LFD f26, 22 * SIZE(BO) |
| LFD f27, 23 * SIZE(BO) |
| |
| addi AO, AO, 4 * SIZE |
| addi BO, BO, 16 * SIZE |
| DCBT(BO, PREB) |
| bdnz LL(32) |
| |
| fadd f0, f1, f0 |
| fadd f4, f5, f4 |
| fadd f8, f9, f8 |
| fadd f12, f13, f12 |
| .align 4 |
| |
| LL(35): |
| lfd f30, ALPHA |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 1 |
| #else |
| addi TEMP, KK, 4 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(38) |
| .align 4 |
| |
| LL(36): |
| FMADD f0, f16, f20, f0 |
| FMADD f4, f16, f21, f4 |
| FMADD f8, f16, f22, f8 |
| FMADD f12, f16, f23, f12 |
| |
| LFD f16, 1 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| addi BO, BO, 4 * SIZE |
| addi AO, AO, 1 * SIZE |
| bdnz LL(36) |
| .align 4 |
| |
| LL(38): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f18, 0 * SIZE(CO2) |
| LFD f20, 0 * SIZE(CO3) |
| LFD f22, 0 * SIZE(CO4) |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f4, f4, f30, f18 |
| FMADD f8, f8, f30, f20 |
| FMADD f12, f12, f30, f22 |
| #else |
| FMUL f0, f0, f30 |
| FMUL f4, f4, f30 |
| FMUL f8, f8, f30 |
| FMUL f12, f12, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f4, 0 * SIZE(CO2) |
| STFD f8, 0 * SIZE(CO3) |
| STFD f12, 0 * SIZE(CO4) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f4, f0 |
| fmr f5, f0 |
| |
| fmr f8, f0 |
| fmr f9, f0 |
| fmr f12, f0 |
| fmr f13, f0 |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -1 |
| #else |
| addi TEMP, TEMP, -4 |
| #endif |
| slwi r0, TEMP, 0 + BASE_SHIFT |
| slwi TEMP, TEMP, 2 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 2 |
| #endif |
| #endif |
| .align 4 |
| |
| |
| LL(39): |
| #if defined(TRMMKERNEL) && !defined(LEFT) |
| addi KK, KK, 4 |
| #endif |
| |
| mr B, BO |
| addic. J, J, -1 |
| bgt LL(10) |
| .align 4 |
| |
| LL(40): |
| mr CO1, C |
| add CO2, C, LDC |
| andi. J, N, 2 |
| ble LL(70) |
| |
| #if defined(TRMMKERNEL) && defined(LEFT) |
| mr KK, OFFSET |
| #endif |
| |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| fmr f4, f0 |
| fmr f5, f0 |
| fmr f6, f0 |
| fmr f7, f0 |
| |
| srawi. I, M, 2 |
| add C, CO2, LDC |
| mr AO, A |
| ble LL(50) |
| .align 4 |
| |
| LL(41): |
| #if defined(TRMMKERNEL) |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 2 + BASE_SHIFT |
| slwi TEMP, KK, 1 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| #endif |
| |
| DCBTST(CO1, PREC) |
| DCBTST(CO2, PREC) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 4 |
| #else |
| addi TEMP, KK, 2 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| |
| #else |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| DCBTST(CO1, PREC) |
| DCBTST(CO2, PREC) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(45) |
| .align 5 |
| |
| LL(42): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| |
| FMADD f4, f16, f21, f4 |
| FMADD f5, f17, f21, f5 |
| FMADD f6, f18, f21, f6 |
| FMADD f7, f19, f21, f7 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| FMADD f0, f16, f22, f0 |
| FMADD f1, f17, f22, f1 |
| FMADD f2, f18, f22, f2 |
| FMADD f3, f19, f22, f3 |
| |
| FMADD f4, f16, f23, f4 |
| FMADD f5, f17, f23, f5 |
| FMADD f6, f18, f23, f6 |
| FMADD f7, f19, f23, f7 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| |
| FMADD f4, f16, f21, f4 |
| FMADD f5, f17, f21, f5 |
| FMADD f6, f18, f21, f6 |
| FMADD f7, f19, f21, f7 |
| |
| LFD f16, 12 * SIZE(AO) |
| LFD f17, 13 * SIZE(AO) |
| LFD f18, 14 * SIZE(AO) |
| LFD f19, 15 * SIZE(AO) |
| |
| FMADD f0, f16, f22, f0 |
| FMADD f1, f17, f22, f1 |
| FMADD f2, f18, f22, f2 |
| FMADD f3, f19, f22, f3 |
| |
| FMADD f4, f16, f23, f4 |
| FMADD f5, f17, f23, f5 |
| FMADD f6, f18, f23, f6 |
| FMADD f7, f19, f23, f7 |
| |
| LFD f16, 16 * SIZE(AO) |
| LFD f17, 17 * SIZE(AO) |
| LFD f18, 18 * SIZE(AO) |
| LFD f19, 19 * SIZE(AO) |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| addi AO, AO, 16 * SIZE |
| addi BO, BO, 8 * SIZE |
| DCBT(BO, PREB) |
| bdnz LL(42) |
| .align 4 |
| |
| LL(45): |
| lfd f30, ALPHA |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 4 |
| #else |
| addi TEMP, KK, 2 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| #else |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| #endif |
| ble+ LL(48) |
| .align 4 |
| |
| LL(46): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| |
| FMADD f4, f16, f21, f4 |
| FMADD f5, f17, f21, f5 |
| FMADD f6, f18, f21, f6 |
| FMADD f7, f19, f21, f7 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f20, 2 * SIZE(BO) |
| LFD f21, 3 * SIZE(BO) |
| |
| addi BO, BO, 2 * SIZE |
| addi AO, AO, 4 * SIZE |
| bdnz LL(46) |
| .align 4 |
| |
| LL(48): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f17, 1 * SIZE(CO1) |
| LFD f18, 2 * SIZE(CO1) |
| LFD f19, 3 * SIZE(CO1) |
| |
| LFD f20, 0 * SIZE(CO2) |
| LFD f21, 1 * SIZE(CO2) |
| LFD f22, 2 * SIZE(CO2) |
| LFD f23, 3 * SIZE(CO2) |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f17 |
| FMADD f2, f2, f30, f18 |
| FMADD f3, f3, f30, f19 |
| |
| FMADD f4, f4, f30, f20 |
| FMADD f5, f5, f30, f21 |
| FMADD f6, f6, f30, f22 |
| FMADD f7, f7, f30, f23 |
| #else |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| FMUL f2, f2, f30 |
| FMUL f3, f3, f30 |
| |
| FMUL f4, f4, f30 |
| FMUL f5, f5, f30 |
| FMUL f6, f6, f30 |
| FMUL f7, f7, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 1 * SIZE(CO1) |
| STFD f2, 2 * SIZE(CO1) |
| STFD f3, 3 * SIZE(CO1) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| STFD f4, 0 * SIZE(CO2) |
| STFD f5, 1 * SIZE(CO2) |
| STFD f6, 2 * SIZE(CO2) |
| STFD f7, 3 * SIZE(CO2) |
| |
| fmr f4, f0 |
| fmr f5, f0 |
| fmr f6, f0 |
| fmr f7, f0 |
| |
| addi CO1, CO1, 4 * SIZE |
| addi CO2, CO2, 4 * SIZE |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -4 |
| #else |
| addi TEMP, TEMP, -2 |
| #endif |
| slwi r0, TEMP, 2 + BASE_SHIFT |
| slwi TEMP, TEMP, 1 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 4 |
| #endif |
| #endif |
| |
| addic. I, I, -1 |
| bgt+ LL(41) |
| .align 4 |
| |
| LL(50): |
| andi. I, M, 2 |
| ble LL(60) |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 1 + BASE_SHIFT |
| slwi TEMP, KK, 1 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| |
| LFD f24, 4 * SIZE(BO) |
| LFD f25, 5 * SIZE(BO) |
| LFD f26, 6 * SIZE(BO) |
| LFD f27, 7 * SIZE(BO) |
| #endif |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 2 |
| #else |
| addi TEMP, KK, 2 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(55) |
| .align 5 |
| |
| LL(52): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f16, f21, f2 |
| FMADD f3, f17, f21, f3 |
| |
| FMADD f4, f18, f22, f4 |
| FMADD f5, f19, f22, f5 |
| FMADD f6, f18, f23, f6 |
| FMADD f7, f19, f23, f7 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| FMADD f0, f16, f24, f0 |
| FMADD f1, f17, f24, f1 |
| FMADD f2, f16, f25, f2 |
| FMADD f3, f17, f25, f3 |
| |
| FMADD f4, f18, f26, f4 |
| FMADD f5, f19, f26, f5 |
| FMADD f6, f18, f27, f6 |
| FMADD f7, f19, f27, f7 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| LFD f24, 12 * SIZE(BO) |
| LFD f25, 13 * SIZE(BO) |
| LFD f26, 14 * SIZE(BO) |
| LFD f27, 15 * SIZE(BO) |
| |
| addi AO, AO, 8 * SIZE |
| addi BO, BO, 8 * SIZE |
| DCBT(BO, PREB) |
| bdnz LL(52) |
| .align 4 |
| |
| LL(55): |
| lfd f30, ALPHA |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 2 |
| #else |
| addi TEMP, KK, 2 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| #endif |
| ble+ LL(58) |
| .align 4 |
| |
| LL(56): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f16, f21, f2 |
| FMADD f3, f17, f21, f3 |
| |
| LFD f16, 2 * SIZE(AO) |
| LFD f17, 3 * SIZE(AO) |
| LFD f20, 2 * SIZE(BO) |
| LFD f21, 3 * SIZE(BO) |
| |
| addi BO, BO, 2 * SIZE |
| addi AO, AO, 2 * SIZE |
| bdnz LL(56) |
| .align 4 |
| |
| LL(58): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f17, 1 * SIZE(CO1) |
| LFD f18, 0 * SIZE(CO2) |
| LFD f19, 1 * SIZE(CO2) |
| |
| FADD f0, f4, f0 |
| FADD f1, f5, f1 |
| FADD f2, f6, f2 |
| FADD f3, f7, f3 |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f17 |
| FMADD f2, f2, f30, f18 |
| FMADD f3, f3, f30, f19 |
| #else |
| FADD f0, f4, f0 |
| FADD f1, f5, f1 |
| FADD f2, f6, f2 |
| FADD f3, f7, f3 |
| |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| FMUL f2, f2, f30 |
| FMUL f3, f3, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 1 * SIZE(CO1) |
| STFD f2, 0 * SIZE(CO2) |
| STFD f3, 1 * SIZE(CO2) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| fmr f4, f0 |
| fmr f5, f0 |
| fmr f6, f0 |
| fmr f7, f0 |
| |
| addi CO1, CO1, 2 * SIZE |
| addi CO2, CO2, 2 * SIZE |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -2 |
| #else |
| addi TEMP, TEMP, -2 |
| #endif |
| slwi r0, TEMP, 1 + BASE_SHIFT |
| slwi TEMP, TEMP, 1 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 2 |
| #endif |
| #endif |
| .align 4 |
| |
| LL(60): |
| andi. I, M, 1 |
| ble LL(69) |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 0 + BASE_SHIFT |
| slwi TEMP, KK, 1 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| |
| LFD f24, 4 * SIZE(BO) |
| LFD f25, 5 * SIZE(BO) |
| LFD f26, 6 * SIZE(BO) |
| LFD f27, 7 * SIZE(BO) |
| #endif |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 1 |
| #else |
| addi TEMP, KK, 2 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| LFD f24, 4 * SIZE(B) |
| LFD f25, 5 * SIZE(B) |
| LFD f26, 6 * SIZE(B) |
| LFD f27, 7 * SIZE(B) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(65) |
| .align 5 |
| |
| LL(62): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f16, f21, f1 |
| FMADD f2, f17, f22, f2 |
| FMADD f3, f17, f23, f3 |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| FMADD f0, f18, f24, f0 |
| FMADD f1, f18, f25, f1 |
| FMADD f2, f19, f26, f2 |
| FMADD f3, f19, f27, f3 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f24, 12 * SIZE(BO) |
| LFD f25, 13 * SIZE(BO) |
| LFD f26, 14 * SIZE(BO) |
| LFD f27, 15 * SIZE(BO) |
| |
| addi AO, AO, 4 * SIZE |
| addi BO, BO, 8 * SIZE |
| bdnz LL(62) |
| .align 4 |
| |
| LL(65): |
| lfd f30, ALPHA |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 1 |
| #else |
| addi TEMP, KK, 2 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(68) |
| .align 4 |
| |
| LL(66): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f16, f21, f1 |
| |
| LFD f16, 1 * SIZE(AO) |
| |
| LFD f20, 2 * SIZE(BO) |
| LFD f21, 3 * SIZE(BO) |
| |
| addi BO, BO, 2 * SIZE |
| addi AO, AO, 1 * SIZE |
| bdnz LL(66) |
| .align 4 |
| |
| LL(68): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f18, 0 * SIZE(CO2) |
| |
| FADD f0, f2, f0 |
| FADD f1, f3, f1 |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f18 |
| #else |
| FADD f0, f2, f0 |
| FADD f1, f3, f1 |
| |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 0 * SIZE(CO2) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f4, f0 |
| fmr f5, f0 |
| |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -1 |
| #else |
| addi TEMP, TEMP, -2 |
| #endif |
| slwi r0, TEMP, 0 + BASE_SHIFT |
| slwi TEMP, TEMP, 1 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 1 |
| #endif |
| #endif |
| .align 4 |
| |
| LL(69): |
| #if defined(TRMMKERNEL) && !defined(LEFT) |
| addi KK, KK, 2 |
| #endif |
| |
| mr B, BO |
| .align 4 |
| |
| LL(70): |
| mr CO1, C |
| andi. J, N, 1 |
| ble LL(999) |
| |
| #if defined(TRMMKERNEL) && defined(LEFT) |
| mr KK, OFFSET |
| #endif |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| srawi. I, M, 2 |
| mr AO, A |
| ble LL(80) |
| .align 4 |
| |
| LL(71): |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 2 + BASE_SHIFT |
| slwi TEMP, KK, 0 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| #endif |
| |
| DCBTST(CO1, PREC) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 4 |
| #else |
| addi TEMP, KK, 1 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| DCBTST(CO1, PREC) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(75) |
| .align 5 |
| |
| LL(72): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| FMADD f0, f16, f21, f0 |
| FMADD f1, f17, f21, f1 |
| FMADD f2, f18, f21, f2 |
| FMADD f3, f19, f21, f3 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| FMADD f0, f16, f22, f0 |
| FMADD f1, f17, f22, f1 |
| FMADD f2, f18, f22, f2 |
| FMADD f3, f19, f22, f3 |
| |
| LFD f16, 12 * SIZE(AO) |
| LFD f17, 13 * SIZE(AO) |
| LFD f18, 14 * SIZE(AO) |
| LFD f19, 15 * SIZE(AO) |
| |
| FMADD f0, f16, f23, f0 |
| FMADD f1, f17, f23, f1 |
| FMADD f2, f18, f23, f2 |
| FMADD f3, f19, f23, f3 |
| |
| LFD f16, 16 * SIZE(AO) |
| LFD f17, 17 * SIZE(AO) |
| LFD f18, 18 * SIZE(AO) |
| LFD f19, 19 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| addi AO, AO, 16 * SIZE |
| addi BO, BO, 4 * SIZE |
| DCBT(BO, PREB) |
| bdnz LL(72) |
| .align 4 |
| |
| LL(75): |
| lfd f30, ALPHA |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 4 |
| #else |
| addi TEMP, KK, 1 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(78) |
| .align 4 |
| |
| LL(76): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f20, f2 |
| FMADD f3, f19, f20, f3 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f20, 1 * SIZE(BO) |
| |
| addi BO, BO, 1 * SIZE |
| addi AO, AO, 4 * SIZE |
| bdnz LL(76) |
| .align 4 |
| |
| LL(78): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f17, 1 * SIZE(CO1) |
| LFD f18, 2 * SIZE(CO1) |
| LFD f19, 3 * SIZE(CO1) |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f17 |
| FMADD f2, f2, f30, f18 |
| FMADD f3, f3, f30, f19 |
| #else |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| FMUL f2, f2, f30 |
| FMUL f3, f3, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 1 * SIZE(CO1) |
| STFD f2, 2 * SIZE(CO1) |
| STFD f3, 3 * SIZE(CO1) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -4 |
| #else |
| addi TEMP, TEMP, -1 |
| #endif |
| slwi r0 , TEMP, 2 + BASE_SHIFT |
| slwi TEMP, TEMP, 0 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 4 |
| #endif |
| #endif |
| |
| addi CO1, CO1, 4 * SIZE |
| addic. I, I, -1 |
| bgt+ LL(71) |
| .align 4 |
| |
| LL(80): |
| andi. I, M, 2 |
| ble LL(90) |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 1 + BASE_SHIFT |
| slwi TEMP, KK, 0 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| #endif |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 2 |
| #else |
| addi TEMP, KK, 1 |
| #endif |
| srawi. TEMP, TEMP, 2 |
| mtspr CTR, TEMP |
| |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| srawi. r0, K, 2 |
| mtspr CTR, r0 |
| mr BO, B |
| |
| #endif |
| ble LL(85) |
| .align 5 |
| |
| LL(82): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| FMADD f2, f18, f21, f2 |
| FMADD f3, f19, f21, f3 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| FMADD f0, f16, f22, f0 |
| FMADD f1, f17, f22, f1 |
| FMADD f2, f18, f23, f2 |
| FMADD f3, f19, f23, f3 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| addi AO, AO, 8 * SIZE |
| addi BO, BO, 4 * SIZE |
| DCBT(BO, PREB) |
| bdnz LL(82) |
| .align 4 |
| |
| LL(85): |
| lfd f30, ALPHA |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 2 |
| #else |
| addi TEMP, KK, 1 |
| #endif |
| andi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| |
| andi. r0, K, 3 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(88) |
| .align 4 |
| |
| LL(86): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f20, f1 |
| |
| LFD f16, 2 * SIZE(AO) |
| LFD f17, 3 * SIZE(AO) |
| LFD f20, 1 * SIZE(BO) |
| |
| addi BO, BO, 1 * SIZE |
| addi AO, AO, 2 * SIZE |
| bdnz LL(86) |
| .align 4 |
| |
| LL(88): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| LFD f17, 1 * SIZE(CO1) |
| |
| FADD f0, f2, f0 |
| FADD f1, f3, f1 |
| |
| FMADD f0, f0, f30, f16 |
| FMADD f1, f1, f30, f17 |
| #else |
| FADD f0, f2, f0 |
| FADD f1, f3, f1 |
| |
| FMUL f0, f0, f30 |
| FMUL f1, f1, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| STFD f1, 1 * SIZE(CO1) |
| |
| lfs f0, FZERO |
| fmr f1, f0 |
| fmr f2, f0 |
| fmr f3, f0 |
| |
| addi CO1, CO1, 2 * SIZE |
| |
| #ifdef TRMMKERNEL |
| #if ( defined(LEFT) && defined(TRANSA)) || \ |
| (!defined(LEFT) && !defined(TRANSA)) |
| sub TEMP, K, KK |
| #ifdef LEFT |
| addi TEMP, TEMP, -2 |
| #else |
| addi TEMP, TEMP, -1 |
| #endif |
| slwi r0 , TEMP, 1 + BASE_SHIFT |
| slwi TEMP, TEMP, 0 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, BO, TEMP |
| #endif |
| |
| #ifdef LEFT |
| addi KK, KK, 2 |
| #endif |
| #endif |
| .align 4 |
| |
| LL(90): |
| andi. I, M, 1 |
| ble LL(999) |
| |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA)) |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| mr BO, B |
| #else |
| slwi r0, KK, 0 + BASE_SHIFT |
| slwi TEMP, KK, 0 + BASE_SHIFT |
| add AO, AO, r0 |
| add BO, B, TEMP |
| |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(BO) |
| LFD f21, 1 * SIZE(BO) |
| LFD f22, 2 * SIZE(BO) |
| LFD f23, 3 * SIZE(BO) |
| #endif |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 1 |
| #else |
| addi TEMP, KK, 1 |
| #endif |
| srawi. TEMP, TEMP, 3 |
| mtspr CTR, TEMP |
| |
| #else |
| LFD f16, 0 * SIZE(AO) |
| LFD f17, 1 * SIZE(AO) |
| LFD f18, 2 * SIZE(AO) |
| LFD f19, 3 * SIZE(AO) |
| |
| LFD f20, 0 * SIZE(B) |
| LFD f21, 1 * SIZE(B) |
| LFD f22, 2 * SIZE(B) |
| LFD f23, 3 * SIZE(B) |
| |
| srawi. r0, K, 3 |
| mtspr CTR, r0 |
| mr BO, B |
| #endif |
| ble LL(95) |
| .align 5 |
| |
| LL(92): |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f21, f1 |
| FMADD f2, f18, f22, f2 |
| FMADD f3, f19, f23, f3 |
| |
| LFD f16, 4 * SIZE(AO) |
| LFD f17, 5 * SIZE(AO) |
| LFD f18, 6 * SIZE(AO) |
| LFD f19, 7 * SIZE(AO) |
| |
| LFD f20, 4 * SIZE(BO) |
| LFD f21, 5 * SIZE(BO) |
| LFD f22, 6 * SIZE(BO) |
| LFD f23, 7 * SIZE(BO) |
| |
| FMADD f0, f16, f20, f0 |
| FMADD f1, f17, f21, f1 |
| FMADD f2, f18, f22, f2 |
| FMADD f3, f19, f23, f3 |
| |
| LFD f16, 8 * SIZE(AO) |
| LFD f17, 9 * SIZE(AO) |
| LFD f18, 10 * SIZE(AO) |
| LFD f19, 11 * SIZE(AO) |
| |
| LFD f20, 8 * SIZE(BO) |
| LFD f21, 9 * SIZE(BO) |
| LFD f22, 10 * SIZE(BO) |
| LFD f23, 11 * SIZE(BO) |
| |
| addi AO, AO, 8 * SIZE |
| addi BO, BO, 8 * SIZE |
| bdnz LL(92) |
| .align 4 |
| |
| LL(95): |
| lfd f30, ALPHA |
| |
| #if defined(TRMMKERNEL) |
| |
| #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA)) |
| sub TEMP, K, KK |
| #elif defined(LEFT) |
| addi TEMP, KK, 1 |
| #else |
| addi TEMP, KK, 1 |
| #endif |
| andi. TEMP, TEMP, 7 |
| mtspr CTR, TEMP |
| |
| #else |
| |
| andi. r0, K, 7 |
| mtspr CTR, r0 |
| |
| #endif |
| ble+ LL(98) |
| .align 4 |
| |
| LL(96): |
| FMADD f0, f16, f20, f0 |
| LFD f16, 1 * SIZE(AO) |
| LFD f20, 1 * SIZE(BO) |
| addi BO, BO, 1 * SIZE |
| addi AO, AO, 1 * SIZE |
| bdnz LL(96) |
| .align 4 |
| |
| LL(98): |
| #ifndef TRMMKERNEL |
| LFD f16, 0 * SIZE(CO1) |
| |
| FADD f0, f1, f0 |
| FADD f2, f3, f2 |
| FADD f0, f2, f0 |
| |
| FMADD f0, f0, f30, f16 |
| #else |
| FADD f0, f1, f0 |
| FADD f2, f3, f2 |
| FADD f0, f2, f0 |
| |
| FMUL f0, f0, f30 |
| #endif |
| |
| STFD f0, 0 * SIZE(CO1) |
| .align 4 |
| |
| LL(999): |
| addi r3, 0, 0 |
| |
| lfd f14, 0(SP) |
| lfd f15, 8(SP) |
| lfd f16, 16(SP) |
| lfd f17, 24(SP) |
| |
| lfd f18, 32(SP) |
| lfd f19, 40(SP) |
| lfd f20, 48(SP) |
| lfd f21, 56(SP) |
| |
| lfd f22, 64(SP) |
| lfd f23, 72(SP) |
| lfd f24, 80(SP) |
| lfd f25, 88(SP) |
| |
| lfd f26, 96(SP) |
| lfd f27, 104(SP) |
| lfd f28, 112(SP) |
| lfd f29, 120(SP) |
| |
| lfd f30, 128(SP) |
| lfd f31, 136(SP) |
| |
| #ifdef __64BIT__ |
| ld r31, 144(SP) |
| ld r30, 152(SP) |
| ld r29, 160(SP) |
| ld r28, 168(SP) |
| ld r27, 176(SP) |
| ld r26, 184(SP) |
| ld r25, 192(SP) |
| ld r24, 200(SP) |
| ld r23, 208(SP) |
| ld r22, 216(SP) |
| ld r21, 224(SP) |
| ld r20, 232(SP) |
| #if defined(TRMMKERNEL) || defined(TRSMKERNEL) |
| ld r19, 240(SP) |
| ld r18, 248(SP) |
| #endif |
| #else |
| lwz r31, 144(SP) |
| lwz r30, 148(SP) |
| lwz r29, 152(SP) |
| lwz r28, 156(SP) |
| lwz r27, 160(SP) |
| lwz r26, 164(SP) |
| lwz r25, 168(SP) |
| lwz r24, 172(SP) |
| lwz r23, 176(SP) |
| lwz r22, 180(SP) |
| lwz r21, 184(SP) |
| lwz r20, 188(SP) |
| #if defined(TRMMKERNEL) || defined(TRSMKERNEL) |
| lwz r19, 192(SP) |
| lwz r18, 196(SP) |
| #endif |
| #endif |
| |
| addi SP, SP, STACKSIZE |
| |
| blr |
| |
| EPILOGUE |
| #endif |
