+++ /dev/null
-#!/usr/bin/env perl
-#
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
-# ====================================================================
-#
-# May 2011
-#
-# The module implements bn_GF2m_mul_2x2 polynomial multiplication
-# used in bn_gf2m.c. It's kind of low-hanging mechanical port from
-# C for the time being... Except that it has two code paths: pure
-# integer code suitable for any ARMv4 and later CPU and NEON code
-# suitable for ARMv7. Pure integer 1x1 multiplication subroutine runs
-# in ~45 cycles on dual-issue core such as Cortex A8, which is ~50%
-# faster than compiler-generated code. For ECDH and ECDSA verify (but
-# not for ECDSA sign) it means 25%-45% improvement depending on key
-# length, more for longer keys. Even though NEON 1x1 multiplication
-# runs in even less cycles, ~30, improvement is measurable only on
-# longer keys. One has to optimize code elsewhere to get NEON glow...
-
-while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
-open STDOUT,">$output";
-
-sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; }
-sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; }
-sub Q() { shift=~m|d([1-3]?[02468])|?"q".($1/2):""; }
-
-$code=<<___;
-#include "arm_arch.h"
-
-.text
-.code 32
-
-#if __ARM_ARCH__>=7
-.fpu neon
-
-.type mul_1x1_neon,%function
-.align 5
-mul_1x1_neon:
- vshl.u64 `&Dlo("q1")`,d16,#8 @ q1-q3 are slided $a
- vmull.p8 `&Q("d0")`,d16,d17 @ a·bb
- vshl.u64 `&Dlo("q2")`,d16,#16
- vmull.p8 q1,`&Dlo("q1")`,d17 @ a<<8·bb
- vshl.u64 `&Dlo("q3")`,d16,#24
- vmull.p8 q2,`&Dlo("q2")`,d17 @ a<<16·bb
- vshr.u64 `&Dlo("q1")`,#8
- vmull.p8 q3,`&Dlo("q3")`,d17 @ a<<24·bb
- vshl.u64 `&Dhi("q1")`,#24
- veor d0,`&Dlo("q1")`
- vshr.u64 `&Dlo("q2")`,#16
- veor d0,`&Dhi("q1")`
- vshl.u64 `&Dhi("q2")`,#16
- veor d0,`&Dlo("q2")`
- vshr.u64 `&Dlo("q3")`,#24
- veor d0,`&Dhi("q2")`
- vshl.u64 `&Dhi("q3")`,#8
- veor d0,`&Dlo("q3")`
- veor d0,`&Dhi("q3")`
- bx lr
-.size mul_1x1_neon,.-mul_1x1_neon
-#endif
-___
-################
-# private interface to mul_1x1_ialu
-#
-$a="r1";
-$b="r0";
-
-($a0,$a1,$a2,$a12,$a4,$a14)=
-($hi,$lo,$t0,$t1, $i0,$i1 )=map("r$_",(4..9),12);
-
-$mask="r12";
-
-$code.=<<___;
-.type mul_1x1_ialu,%function
-.align 5
-mul_1x1_ialu:
- mov $a0,#0
- bic $a1,$a,#3<<30 @ a1=a&0x3fffffff
- str $a0,[sp,#0] @ tab[0]=0
- add $a2,$a1,$a1 @ a2=a1<<1
- str $a1,[sp,#4] @ tab[1]=a1
- eor $a12,$a1,$a2 @ a1^a2
- str $a2,[sp,#8] @ tab[2]=a2
- mov $a4,$a1,lsl#2 @ a4=a1<<2
- str $a12,[sp,#12] @ tab[3]=a1^a2
- eor $a14,$a1,$a4 @ a1^a4
- str $a4,[sp,#16] @ tab[4]=a4
- eor $a0,$a2,$a4 @ a2^a4
- str $a14,[sp,#20] @ tab[5]=a1^a4
- eor $a12,$a12,$a4 @ a1^a2^a4
- str $a0,[sp,#24] @ tab[6]=a2^a4
- and $i0,$mask,$b,lsl#2
- str $a12,[sp,#28] @ tab[7]=a1^a2^a4
-
- and $i1,$mask,$b,lsr#1
- ldr $lo,[sp,$i0] @ tab[b & 0x7]
- and $i0,$mask,$b,lsr#4
- ldr $t1,[sp,$i1] @ tab[b >> 3 & 0x7]
- and $i1,$mask,$b,lsr#7
- ldr $t0,[sp,$i0] @ tab[b >> 6 & 0x7]
- eor $lo,$lo,$t1,lsl#3 @ stall
- mov $hi,$t1,lsr#29
- ldr $t1,[sp,$i1] @ tab[b >> 9 & 0x7]
-
- and $i0,$mask,$b,lsr#10
- eor $lo,$lo,$t0,lsl#6
- eor $hi,$hi,$t0,lsr#26
- ldr $t0,[sp,$i0] @ tab[b >> 12 & 0x7]
-
- and $i1,$mask,$b,lsr#13
- eor $lo,$lo,$t1,lsl#9
- eor $hi,$hi,$t1,lsr#23
- ldr $t1,[sp,$i1] @ tab[b >> 15 & 0x7]
-
- and $i0,$mask,$b,lsr#16
- eor $lo,$lo,$t0,lsl#12
- eor $hi,$hi,$t0,lsr#20
- ldr $t0,[sp,$i0] @ tab[b >> 18 & 0x7]
-
- and $i1,$mask,$b,lsr#19
- eor $lo,$lo,$t1,lsl#15
- eor $hi,$hi,$t1,lsr#17
- ldr $t1,[sp,$i1] @ tab[b >> 21 & 0x7]
-
- and $i0,$mask,$b,lsr#22
- eor $lo,$lo,$t0,lsl#18
- eor $hi,$hi,$t0,lsr#14
- ldr $t0,[sp,$i0] @ tab[b >> 24 & 0x7]
-
- and $i1,$mask,$b,lsr#25
- eor $lo,$lo,$t1,lsl#21
- eor $hi,$hi,$t1,lsr#11
- ldr $t1,[sp,$i1] @ tab[b >> 27 & 0x7]
-
- tst $a,#1<<30
- and $i0,$mask,$b,lsr#28
- eor $lo,$lo,$t0,lsl#24
- eor $hi,$hi,$t0,lsr#8
- ldr $t0,[sp,$i0] @ tab[b >> 30 ]
-
- eorne $lo,$lo,$b,lsl#30
- eorne $hi,$hi,$b,lsr#2
- tst $a,#1<<31
- eor $lo,$lo,$t1,lsl#27
- eor $hi,$hi,$t1,lsr#5
- eorne $lo,$lo,$b,lsl#31
- eorne $hi,$hi,$b,lsr#1
- eor $lo,$lo,$t0,lsl#30
- eor $hi,$hi,$t0,lsr#2
-
- mov pc,lr
-.size mul_1x1_ialu,.-mul_1x1_ialu
-___
-################
-# void bn_GF2m_mul_2x2(BN_ULONG *r,
-# BN_ULONG a1,BN_ULONG a0,
-# BN_ULONG b1,BN_ULONG b0); # r[3..0]=a1a0·b1b0
-
-($A1,$B1,$A0,$B0,$A1B1,$A0B0)=map("d$_",(18..23));
-
-$code.=<<___;
-.global bn_GF2m_mul_2x2
-.type bn_GF2m_mul_2x2,%function
-.align 5
-bn_GF2m_mul_2x2:
-#if __ARM_ARCH__>=7
- ldr r12,.LOPENSSL_armcap
-.Lpic: ldr r12,[pc,r12]
- tst r12,#1
- beq .Lialu
-
- veor $A1,$A1
- vmov $B1,r3,r3 @ two copies of b1
- vmov.32 ${A1}[0],r1 @ a1
-
- veor $A0,$A0
- vld1.32 ${B0}[],[sp,:32] @ two copies of b0
- vmov.32 ${A0}[0],r2 @ a0
- mov r12,lr
-
- vmov d16,$A1
- vmov d17,$B1
- bl mul_1x1_neon @ a1·b1
- vmov $A1B1,d0
-
- vmov d16,$A0
- vmov d17,$B0
- bl mul_1x1_neon @ a0·b0
- vmov $A0B0,d0
-
- veor d16,$A0,$A1
- veor d17,$B0,$B1
- veor $A0,$A0B0,$A1B1
- bl mul_1x1_neon @ (a0+a1)·(b0+b1)
-
- veor d0,$A0 @ (a0+a1)·(b0+b1)-a0·b0-a1·b1
- vshl.u64 d1,d0,#32
- vshr.u64 d0,d0,#32
- veor $A0B0,d1
- veor $A1B1,d0
- vst1.32 {${A0B0}[0]},[r0,:32]!
- vst1.32 {${A0B0}[1]},[r0,:32]!
- vst1.32 {${A1B1}[0]},[r0,:32]!
- vst1.32 {${A1B1}[1]},[r0,:32]
- bx r12
-.align 4
-.Lialu:
-#endif
-___
-$ret="r10"; # reassigned 1st argument
-$code.=<<___;
- stmdb sp!,{r4-r10,lr}
- mov $ret,r0 @ reassign 1st argument
- mov $b,r3 @ $b=b1
- ldr r3,[sp,#32] @ load b0
- mov $mask,#7<<2
- sub sp,sp,#32 @ allocate tab[8]
-
- bl mul_1x1_ialu @ a1·b1
- str $lo,[$ret,#8]
- str $hi,[$ret,#12]
-
- eor $b,$b,r3 @ flip b0 and b1
- eor $a,$a,r2 @ flip a0 and a1
- eor r3,r3,$b
- eor r2,r2,$a
- eor $b,$b,r3
- eor $a,$a,r2
- bl mul_1x1_ialu @ a0·b0
- str $lo,[$ret]
- str $hi,[$ret,#4]
-
- eor $a,$a,r2
- eor $b,$b,r3
- bl mul_1x1_ialu @ (a1+a0)·(b1+b0)
-___
-@r=map("r$_",(6..9));
-$code.=<<___;
- ldmia $ret,{@r[0]-@r[3]}
- eor $lo,$lo,$hi
- eor $hi,$hi,@r[1]
- eor $lo,$lo,@r[0]
- eor $hi,$hi,@r[2]
- eor $lo,$lo,@r[3]
- eor $hi,$hi,@r[3]
- str $hi,[$ret,#8]
- eor $lo,$lo,$hi
- add sp,sp,#32 @ destroy tab[8]
- str $lo,[$ret,#4]
-
-#if __ARM_ARCH__>=5
- ldmia sp!,{r4-r10,pc}
-#else
- ldmia sp!,{r4-r10,lr}
- tst lr,#1
- moveq pc,lr @ be binary compatible with V4, yet
- bx lr @ interoperable with Thumb ISA:-)
-#endif
-.size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2
-#if __ARM_ARCH__>=7
-.align 5
-.LOPENSSL_armcap:
-.word OPENSSL_armcap_P-(.Lpic+8)
-#endif
-.asciz "GF(2^m) Multiplication for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
-.align 5
-
-.comm OPENSSL_armcap_P,4,4
-___
-
-$code =~ s/\`([^\`]*)\`/eval $1/gem;
-$code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4
-print $code;
-close STDOUT; # enforce flush
+++ /dev/null
-#!/usr/bin/env perl
-#
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
-# ====================================================================
-#
-# May 2011
-#
-# The module implements bn_GF2m_mul_2x2 polynomial multiplication used
-# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
-# the time being... Except that it has three code paths: pure integer
-# code suitable for any x86 CPU, MMX code suitable for PIII and later
-# and PCLMULQDQ suitable for Westmere and later. Improvement varies
-# from one benchmark and µ-arch to another. Below are interval values
-# for 163- and 571-bit ECDH benchmarks relative to compiler-generated
-# code:
-#
-# PIII 16%-30%
-# P4 12%-12%
-# Opteron 18%-40%
-# Core2 19%-44%
-# Atom 38%-64%
-# Westmere 53%-121%(PCLMULQDQ)/20%-32%(MMX)
-# Sandy Bridge 72%-127%(PCLMULQDQ)/27%-23%(MMX)
-#
-# Note that above improvement coefficients are not coefficients for
-# bn_GF2m_mul_2x2 itself. For example 120% ECDH improvement is result
-# of bn_GF2m_mul_2x2 being >4x faster. As it gets faster, benchmark
-# is more and more dominated by other subroutines, most notably by
-# BN_GF2m_mod[_mul]_arr...
-
-$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
-push(@INC,"${dir}","${dir}../../perlasm");
-require "x86asm.pl";
-
-&asm_init($ARGV[0],$0,$x86only = $ARGV[$#ARGV] eq "386");
-
-$sse2=0;
-for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
-
-&external_label("OPENSSL_ia32cap_P") if ($sse2);
-
-$a="eax";
-$b="ebx";
-($a1,$a2,$a4)=("ecx","edx","ebp");
-
-$R="mm0";
-@T=("mm1","mm2");
-($A,$B,$B30,$B31)=("mm2","mm3","mm4","mm5");
-@i=("esi","edi");
-
- if (!$x86only) {
-&function_begin_B("_mul_1x1_mmx");
- &sub ("esp",32+4);
- &mov ($a1,$a);
- &lea ($a2,&DWP(0,$a,$a));
- &and ($a1,0x3fffffff);
- &lea ($a4,&DWP(0,$a2,$a2));
- &mov (&DWP(0*4,"esp"),0);
- &and ($a2,0x7fffffff);
- &movd ($A,$a);
- &movd ($B,$b);
- &mov (&DWP(1*4,"esp"),$a1); # a1
- &xor ($a1,$a2); # a1^a2
- &pxor ($B31,$B31);
- &pxor ($B30,$B30);
- &mov (&DWP(2*4,"esp"),$a2); # a2
- &xor ($a2,$a4); # a2^a4
- &mov (&DWP(3*4,"esp"),$a1); # a1^a2
- &pcmpgtd($B31,$A); # broadcast 31st bit
- &paddd ($A,$A); # $A<<=1
- &xor ($a1,$a2); # a1^a4=a1^a2^a2^a4
- &mov (&DWP(4*4,"esp"),$a4); # a4
- &xor ($a4,$a2); # a2=a4^a2^a4
- &pand ($B31,$B);
- &pcmpgtd($B30,$A); # broadcast 30th bit
- &mov (&DWP(5*4,"esp"),$a1); # a1^a4
- &xor ($a4,$a1); # a1^a2^a4
- &psllq ($B31,31);
- &pand ($B30,$B);
- &mov (&DWP(6*4,"esp"),$a2); # a2^a4
- &mov (@i[0],0x7);
- &mov (&DWP(7*4,"esp"),$a4); # a1^a2^a4
- &mov ($a4,@i[0]);
- &and (@i[0],$b);
- &shr ($b,3);
- &mov (@i[1],$a4);
- &psllq ($B30,30);
- &and (@i[1],$b);
- &shr ($b,3);
- &movd ($R,&DWP(0,"esp",@i[0],4));
- &mov (@i[0],$a4);
- &and (@i[0],$b);
- &shr ($b,3);
- for($n=1;$n<9;$n++) {
- &movd (@T[1],&DWP(0,"esp",@i[1],4));
- &mov (@i[1],$a4);
- &psllq (@T[1],3*$n);
- &and (@i[1],$b);
- &shr ($b,3);
- &pxor ($R,@T[1]);
-
- push(@i,shift(@i)); push(@T,shift(@T));
- }
- &movd (@T[1],&DWP(0,"esp",@i[1],4));
- &pxor ($R,$B30);
- &psllq (@T[1],3*$n++);
- &pxor ($R,@T[1]);
-
- &movd (@T[0],&DWP(0,"esp",@i[0],4));
- &pxor ($R,$B31);
- &psllq (@T[0],3*$n);
- &add ("esp",32+4);
- &pxor ($R,@T[0]);
- &ret ();
-&function_end_B("_mul_1x1_mmx");
- }
-
-($lo,$hi)=("eax","edx");
-@T=("ecx","ebp");
-
-&function_begin_B("_mul_1x1_ialu");
- &sub ("esp",32+4);
- &mov ($a1,$a);
- &lea ($a2,&DWP(0,$a,$a));
- &lea ($a4,&DWP(0,"",$a,4));
- &and ($a1,0x3fffffff);
- &lea (@i[1],&DWP(0,$lo,$lo));
- &sar ($lo,31); # broadcast 31st bit
- &mov (&DWP(0*4,"esp"),0);
- &and ($a2,0x7fffffff);
- &mov (&DWP(1*4,"esp"),$a1); # a1
- &xor ($a1,$a2); # a1^a2
- &mov (&DWP(2*4,"esp"),$a2); # a2
- &xor ($a2,$a4); # a2^a4
- &mov (&DWP(3*4,"esp"),$a1); # a1^a2
- &xor ($a1,$a2); # a1^a4=a1^a2^a2^a4
- &mov (&DWP(4*4,"esp"),$a4); # a4
- &xor ($a4,$a2); # a2=a4^a2^a4
- &mov (&DWP(5*4,"esp"),$a1); # a1^a4
- &xor ($a4,$a1); # a1^a2^a4
- &sar (@i[1],31); # broadcast 30th bit
- &and ($lo,$b);
- &mov (&DWP(6*4,"esp"),$a2); # a2^a4
- &and (@i[1],$b);
- &mov (&DWP(7*4,"esp"),$a4); # a1^a2^a4
- &mov ($hi,$lo);
- &shl ($lo,31);
- &mov (@T[0],@i[1]);
- &shr ($hi,1);
-
- &mov (@i[0],0x7);
- &shl (@i[1],30);
- &and (@i[0],$b);
- &shr (@T[0],2);
- &xor ($lo,@i[1]);
-
- &shr ($b,3);
- &mov (@i[1],0x7); # 5-byte instruction!?
- &and (@i[1],$b);
- &shr ($b,3);
- &xor ($hi,@T[0]);
- &xor ($lo,&DWP(0,"esp",@i[0],4));
- &mov (@i[0],0x7);
- &and (@i[0],$b);
- &shr ($b,3);
- for($n=1;$n<9;$n++) {
- &mov (@T[1],&DWP(0,"esp",@i[1],4));
- &mov (@i[1],0x7);
- &mov (@T[0],@T[1]);
- &shl (@T[1],3*$n);
- &and (@i[1],$b);
- &shr (@T[0],32-3*$n);
- &xor ($lo,@T[1]);
- &shr ($b,3);
- &xor ($hi,@T[0]);
-
- push(@i,shift(@i)); push(@T,shift(@T));
- }
- &mov (@T[1],&DWP(0,"esp",@i[1],4));
- &mov (@T[0],@T[1]);
- &shl (@T[1],3*$n);
- &mov (@i[1],&DWP(0,"esp",@i[0],4));
- &shr (@T[0],32-3*$n); $n++;
- &mov (@i[0],@i[1]);
- &xor ($lo,@T[1]);
- &shl (@i[1],3*$n);
- &xor ($hi,@T[0]);
- &shr (@i[0],32-3*$n);
- &xor ($lo,@i[1]);
- &xor ($hi,@i[0]);
-
- &add ("esp",32+4);
- &ret ();
-&function_end_B("_mul_1x1_ialu");
-
-# void bn_GF2m_mul_2x2(BN_ULONG *r, BN_ULONG a1, BN_ULONG a0, BN_ULONG b1, BN_ULONG b0);
-&function_begin_B("bn_GF2m_mul_2x2");
-if (!$x86only) {
- &picsetup("edx");
- &picsymbol("edx", "OPENSSL_ia32cap_P", "edx");
- &mov ("eax",&DWP(0,"edx"));
- &mov ("edx",&DWP(4,"edx"));
- &test ("eax","\$IA32CAP_MASK0_MMX"); # check MMX bit
- &jz (&label("ialu"));
-if ($sse2) {
- &test ("eax","\$IA32CAP_MASK0_FXSR"); # check FXSR bit
- &jz (&label("mmx"));
- &test ("edx","\$IA32CAP_MASK1_PCLMUL"); # check PCLMULQDQ bit
- &jz (&label("mmx"));
-
- &movups ("xmm0",&QWP(8,"esp"));
- &shufps ("xmm0","xmm0",0b10110001);
- &pclmulqdq ("xmm0","xmm0",1);
- &mov ("eax",&DWP(4,"esp"));
- &movups (&QWP(0,"eax"),"xmm0");
- &ret ();
-
-&set_label("mmx",16);
-}
- &push ("ebp");
- &push ("ebx");
- &push ("esi");
- &push ("edi");
- &mov ($a,&wparam(1));
- &mov ($b,&wparam(3));
- &call ("_mul_1x1_mmx"); # a1·b1
- &movq ("mm7",$R);
-
- &mov ($a,&wparam(2));
- &mov ($b,&wparam(4));
- &call ("_mul_1x1_mmx"); # a0·b0
- &movq ("mm6",$R);
-
- &mov ($a,&wparam(1));
- &mov ($b,&wparam(3));
- &xor ($a,&wparam(2));
- &xor ($b,&wparam(4));
- &call ("_mul_1x1_mmx"); # (a0+a1)·(b0+b1)
- &pxor ($R,"mm7");
- &mov ($a,&wparam(0));
- &pxor ($R,"mm6"); # (a0+a1)·(b0+b1)-a1·b1-a0·b0
-
- &movq ($A,$R);
- &psllq ($R,32);
- &pop ("edi");
- &psrlq ($A,32);
- &pop ("esi");
- &pxor ($R,"mm6");
- &pop ("ebx");
- &pxor ($A,"mm7");
- &movq (&QWP(0,$a),$R);
- &pop ("ebp");
- &movq (&QWP(8,$a),$A);
- &emms ();
- &ret ();
-&set_label("ialu",16);
-}
- &push ("ebp");
- &push ("ebx");
- &push ("esi");
- &push ("edi");
- &stack_push(4+1);
-
- &mov ($a,&wparam(1));
- &mov ($b,&wparam(3));
- &call ("_mul_1x1_ialu"); # a1·b1
- &mov (&DWP(8,"esp"),$lo);
- &mov (&DWP(12,"esp"),$hi);
-
- &mov ($a,&wparam(2));
- &mov ($b,&wparam(4));
- &call ("_mul_1x1_ialu"); # a0·b0
- &mov (&DWP(0,"esp"),$lo);
- &mov (&DWP(4,"esp"),$hi);
-
- &mov ($a,&wparam(1));
- &mov ($b,&wparam(3));
- &xor ($a,&wparam(2));
- &xor ($b,&wparam(4));
- &call ("_mul_1x1_ialu"); # (a0+a1)·(b0+b1)
-
- &mov ("ebp",&wparam(0));
- @r=("ebx","ecx","edi","esi");
- &mov (@r[0],&DWP(0,"esp"));
- &mov (@r[1],&DWP(4,"esp"));
- &mov (@r[2],&DWP(8,"esp"));
- &mov (@r[3],&DWP(12,"esp"));
-
- &xor ($lo,$hi);
- &xor ($hi,@r[1]);
- &xor ($lo,@r[0]);
- &mov (&DWP(0,"ebp"),@r[0]);
- &xor ($hi,@r[2]);
- &mov (&DWP(12,"ebp"),@r[3]);
- &xor ($lo,@r[3]);
- &stack_pop(4+1);
- &xor ($hi,@r[3]);
- &pop ("edi");
- &xor ($lo,$hi);
- &pop ("esi");
- &mov (&DWP(8,"ebp"),$hi);
- &pop ("ebx");
- &mov (&DWP(4,"ebp"),$lo);
- &pop ("ebp");
- &ret ();
-&function_end_B("bn_GF2m_mul_2x2");
-
-&asm_finish();
+++ /dev/null
-#!/usr/bin/env perl
-#
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see http://www.openssl.org/~appro/cryptogams/.
-# ====================================================================
-#
-# May 2011
-#
-# The module implements bn_GF2m_mul_2x2 polynomial multiplication used
-# in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
-# the time being... Except that it has two code paths: code suitable
-# for any x86_64 CPU and PCLMULQDQ one suitable for Westmere and
-# later. Improvement varies from one benchmark and µ-arch to another.
-# Vanilla code path is at most 20% faster than compiler-generated code
-# [not very impressive], while PCLMULQDQ - whole 85%-160% better on
-# 163- and 571-bit ECDH benchmarks on Intel CPUs. Keep in mind that
-# these coefficients are not ones for bn_GF2m_mul_2x2 itself, as not
-# all CPU time is burnt in it...
-
-$flavour = shift;
-$output = shift;
-if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
-
-$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
-
-$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
-( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
-( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
-die "can't locate x86_64-xlate.pl";
-
-open OUT,"| \"$^X\" $xlate $flavour $output";
-*STDOUT=*OUT;
-
-($lo,$hi)=("%rax","%rdx"); $a=$lo;
-($i0,$i1)=("%rsi","%rdi");
-($t0,$t1)=("%rbx","%rcx");
-($b,$mask)=("%rbp","%r8");
-($a1,$a2,$a4,$a8,$a12,$a48)=map("%r$_",(9..15));
-($R,$Tx)=("%xmm0","%xmm1");
-
-$code.=<<___;
-.text
-
-.type _mul_1x1,\@abi-omnipotent
-.align 16
-_mul_1x1:
- sub \$128+8,%rsp
- mov \$-1,$a1
- lea ($a,$a),$i0
- shr \$3,$a1
- lea (,$a,4),$i1
- and $a,$a1 # a1=a&0x1fffffffffffffff
- lea (,$a,8),$a8
- sar \$63,$a # broadcast 63rd bit
- lea ($a1,$a1),$a2
- sar \$63,$i0 # broadcast 62nd bit
- lea (,$a1,4),$a4
- and $b,$a
- sar \$63,$i1 # broadcast 61st bit
- mov $a,$hi # $a is $lo
- shl \$63,$lo
- and $b,$i0
- shr \$1,$hi
- mov $i0,$t1
- shl \$62,$i0
- and $b,$i1
- shr \$2,$t1
- xor $i0,$lo
- mov $i1,$t0
- shl \$61,$i1
- xor $t1,$hi
- shr \$3,$t0
- xor $i1,$lo
- xor $t0,$hi
-
- mov $a1,$a12
- movq \$0,0(%rsp) # tab[0]=0
- xor $a2,$a12 # a1^a2
- mov $a1,8(%rsp) # tab[1]=a1
- mov $a4,$a48
- mov $a2,16(%rsp) # tab[2]=a2
- xor $a8,$a48 # a4^a8
- mov $a12,24(%rsp) # tab[3]=a1^a2
-
- xor $a4,$a1
- mov $a4,32(%rsp) # tab[4]=a4
- xor $a4,$a2
- mov $a1,40(%rsp) # tab[5]=a1^a4
- xor $a4,$a12
- mov $a2,48(%rsp) # tab[6]=a2^a4
- xor $a48,$a1 # a1^a4^a4^a8=a1^a8
- mov $a12,56(%rsp) # tab[7]=a1^a2^a4
- xor $a48,$a2 # a2^a4^a4^a8=a1^a8
-
- mov $a8,64(%rsp) # tab[8]=a8
- xor $a48,$a12 # a1^a2^a4^a4^a8=a1^a2^a8
- mov $a1,72(%rsp) # tab[9]=a1^a8
- xor $a4,$a1 # a1^a8^a4
- mov $a2,80(%rsp) # tab[10]=a2^a8
- xor $a4,$a2 # a2^a8^a4
- mov $a12,88(%rsp) # tab[11]=a1^a2^a8
-
- xor $a4,$a12 # a1^a2^a8^a4
- mov $a48,96(%rsp) # tab[12]=a4^a8
- mov $mask,$i0
- mov $a1,104(%rsp) # tab[13]=a1^a4^a8
- and $b,$i0
- mov $a2,112(%rsp) # tab[14]=a2^a4^a8
- shr \$4,$b
- mov $a12,120(%rsp) # tab[15]=a1^a2^a4^a8
- mov $mask,$i1
- and $b,$i1
- shr \$4,$b
-
- movq (%rsp,$i0,8),$R # half of calculations is done in SSE2
- mov $mask,$i0
- and $b,$i0
- shr \$4,$b
-___
- for ($n=1;$n<8;$n++) {
- $code.=<<___;
- mov (%rsp,$i1,8),$t1
- mov $mask,$i1
- mov $t1,$t0
- shl \$`8*$n-4`,$t1
- and $b,$i1
- movq (%rsp,$i0,8),$Tx
- shr \$`64-(8*$n-4)`,$t0
- xor $t1,$lo
- pslldq \$$n,$Tx
- mov $mask,$i0
- shr \$4,$b
- xor $t0,$hi
- and $b,$i0
- shr \$4,$b
- pxor $Tx,$R
-___
- }
-$code.=<<___;
- mov (%rsp,$i1,8),$t1
- mov $t1,$t0
- shl \$`8*$n-4`,$t1
- movd $R,$i0
- shr \$`64-(8*$n-4)`,$t0
- xor $t1,$lo
- psrldq \$8,$R
- xor $t0,$hi
- movd $R,$i1
- xor $i0,$lo
- xor $i1,$hi
-
- add \$128+8,%rsp
- ret
-.Lend_mul_1x1:
-.size _mul_1x1,.-_mul_1x1
-___
-
-($rp,$a1,$a0,$b1,$b0) = $win64? ("%rcx","%rdx","%r8", "%r9","%r10") : # Win64 order
- ("%rdi","%rsi","%rdx","%rcx","%r8"); # Unix order
-
-$code.=<<___;
-.extern OPENSSL_ia32cap_P
-.hidden OPENSSL_ia32cap_P
-.globl bn_GF2m_mul_2x2
-.type bn_GF2m_mul_2x2,\@abi-omnipotent
-.align 16
-bn_GF2m_mul_2x2:
- mov OPENSSL_ia32cap_P+4(%rip),%eax
- bt \$IA32CAP_BIT1_PCLMUL,%eax
- jnc .Lvanilla_mul_2x2
-
- movd $a1,%xmm0
- movd $b1,%xmm1
- movd $a0,%xmm2
-___
-$code.=<<___ if ($win64);
- movq 40(%rsp),%xmm3
-___
-$code.=<<___ if (!$win64);
- movd $b0,%xmm3
-___
-$code.=<<___;
- movdqa %xmm0,%xmm4
- movdqa %xmm1,%xmm5
- pclmulqdq \$0,%xmm1,%xmm0 # a1·b1
- pxor %xmm2,%xmm4
- pxor %xmm3,%xmm5
- pclmulqdq \$0,%xmm3,%xmm2 # a0·b0
- pclmulqdq \$0,%xmm5,%xmm4 # (a0+a1)·(b0+b1)
- xorps %xmm0,%xmm4
- xorps %xmm2,%xmm4 # (a0+a1)·(b0+b1)-a0·b0-a1·b1
- movdqa %xmm4,%xmm5
- pslldq \$8,%xmm4
- psrldq \$8,%xmm5
- pxor %xmm4,%xmm2
- pxor %xmm5,%xmm0
- movdqu %xmm2,0($rp)
- movdqu %xmm0,16($rp)
- ret
-
-.align 16
-.Lvanilla_mul_2x2:
- lea -8*17(%rsp),%rsp
-___
-$code.=<<___ if ($win64);
- mov `8*17+40`(%rsp),$b0
- mov %rdi,8*15(%rsp)
- mov %rsi,8*16(%rsp)
-___
-$code.=<<___;
- mov %r14,8*10(%rsp)
- mov %r13,8*11(%rsp)
- mov %r12,8*12(%rsp)
- mov %rbp,8*13(%rsp)
- mov %rbx,8*14(%rsp)
-.Lbody_mul_2x2:
- mov $rp,32(%rsp) # save the arguments
- mov $a1,40(%rsp)
- mov $a0,48(%rsp)
- mov $b1,56(%rsp)
- mov $b0,64(%rsp)
-
- mov \$0xf,$mask
- mov $a1,$a
- mov $b1,$b
- call _mul_1x1 # a1·b1
- mov $lo,16(%rsp)
- mov $hi,24(%rsp)
-
- mov 48(%rsp),$a
- mov 64(%rsp),$b
- call _mul_1x1 # a0·b0
- mov $lo,0(%rsp)
- mov $hi,8(%rsp)
-
- mov 40(%rsp),$a
- mov 56(%rsp),$b
- xor 48(%rsp),$a
- xor 64(%rsp),$b
- call _mul_1x1 # (a0+a1)·(b0+b1)
-___
- @r=("%rbx","%rcx","%rdi","%rsi");
-$code.=<<___;
- mov 0(%rsp),@r[0]
- mov 8(%rsp),@r[1]
- mov 16(%rsp),@r[2]
- mov 24(%rsp),@r[3]
- mov 32(%rsp),%rbp
-
- xor $hi,$lo
- xor @r[1],$hi
- xor @r[0],$lo
- mov @r[0],0(%rbp)
- xor @r[2],$hi
- mov @r[3],24(%rbp)
- xor @r[3],$lo
- xor @r[3],$hi
- xor $hi,$lo
- mov $hi,16(%rbp)
- mov $lo,8(%rbp)
-
- mov 8*10(%rsp),%r14
- mov 8*11(%rsp),%r13
- mov 8*12(%rsp),%r12
- mov 8*13(%rsp),%rbp
- mov 8*14(%rsp),%rbx
-___
-$code.=<<___ if ($win64);
- mov 8*15(%rsp),%rdi
- mov 8*16(%rsp),%rsi
-___
-$code.=<<___;
- lea 8*17(%rsp),%rsp
- ret
-.Lend_mul_2x2:
-.size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2
-.align 16
-___
-
-# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
-# CONTEXT *context,DISPATCHER_CONTEXT *disp)
-if ($win64) {
-$rec="%rcx";
-$frame="%rdx";
-$context="%r8";
-$disp="%r9";
-
-$code.=<<___;
-.extern __imp_RtlVirtualUnwind
-
-.type se_handler,\@abi-omnipotent
-.align 16
-se_handler:
- push %rsi
- push %rdi
- push %rbx
- push %rbp
- push %r12
- push %r13
- push %r14
- push %r15
- pushfq
- sub \$64,%rsp
-
- mov 152($context),%rax # pull context->Rsp
- mov 248($context),%rbx # pull context->Rip
-
- lea .Lbody_mul_2x2(%rip),%r10
- cmp %r10,%rbx # context->Rip<"prologue" label
- jb .Lin_prologue
-
- mov 8*10(%rax),%r14 # mimic epilogue
- mov 8*11(%rax),%r13
- mov 8*12(%rax),%r12
- mov 8*13(%rax),%rbp
- mov 8*14(%rax),%rbx
- mov 8*15(%rax),%rdi
- mov 8*16(%rax),%rsi
-
- mov %rbx,144($context) # restore context->Rbx
- mov %rbp,160($context) # restore context->Rbp
- mov %rsi,168($context) # restore context->Rsi
- mov %rdi,176($context) # restore context->Rdi
- mov %r12,216($context) # restore context->R12
- mov %r13,224($context) # restore context->R13
- mov %r14,232($context) # restore context->R14
-
-.Lin_prologue:
- lea 8*17(%rax),%rax
- mov %rax,152($context) # restore context->Rsp
-
- mov 40($disp),%rdi # disp->ContextRecord
- mov $context,%rsi # context
- mov \$154,%ecx # sizeof(CONTEXT)
- .long 0xa548f3fc # cld; rep movsq
-
- mov $disp,%rsi
- xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
- mov 8(%rsi),%rdx # arg2, disp->ImageBase
- mov 0(%rsi),%r8 # arg3, disp->ControlPc
- mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
- mov 40(%rsi),%r10 # disp->ContextRecord
- lea 56(%rsi),%r11 # &disp->HandlerData
- lea 24(%rsi),%r12 # &disp->EstablisherFrame
- mov %r10,32(%rsp) # arg5
- mov %r11,40(%rsp) # arg6
- mov %r12,48(%rsp) # arg7
- mov %rcx,56(%rsp) # arg8, (NULL)
- call *__imp_RtlVirtualUnwind(%rip)
-
- mov \$1,%eax # ExceptionContinueSearch
- add \$64,%rsp
- popfq
- pop %r15
- pop %r14
- pop %r13
- pop %r12
- pop %rbp
- pop %rbx
- pop %rdi
- pop %rsi
- ret
-.size se_handler,.-se_handler
-
-.section .pdata
-.align 4
- .rva _mul_1x1
- .rva .Lend_mul_1x1
- .rva .LSEH_info_1x1
-
- .rva .Lvanilla_mul_2x2
- .rva .Lend_mul_2x2
- .rva .LSEH_info_2x2
-.section .xdata
-.align 8
-.LSEH_info_1x1:
- .byte 0x01,0x07,0x02,0x00
- .byte 0x07,0x01,0x11,0x00 # sub rsp,128+8
-.LSEH_info_2x2:
- .byte 9,0,0,0
- .rva se_handler
-___
-}
-
-$code =~ s/\`([^\`]*)\`/eval($1)/gem;
-print $code;
-close STDOUT;
projects / openbsd / commitdiff