Preserve comments in perlasm. This is needed so we can have copyright comments in generated files. Bug: 503700354 Change-Id: I82683c6670375e278afdec2c71fb992d6a6a6964 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/98508 Commit-Queue: Adam Langley <agl@google.com> Reviewed-by: Adam Langley <agl@google.com> Auto-Submit: Rudolf Polzer <rpolzer@google.com>
diff --git a/crypto/fipsmodule/bn/asm/x86_64-mont5.pl b/crypto/fipsmodule/bn/asm/x86_64-mont5.pl index 35911ab..59ea195 100755 --- a/crypto/fipsmodule/bn/asm/x86_64-mont5.pl +++ b/crypto/fipsmodule/bn/asm/x86_64-mont5.pl
@@ -3479,7 +3479,7 @@ # I can't trust assembler to use specific encoding:-( .byte 0x4c,0x8d,0x14,0x24 #lea (%rsp),%r10 .cfi_def_cfa_register %r10 - .byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 #sub $0x108,%rsp + .byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 #sub \$0x108,%rsp lea .Linc(%rip),%rax and \$-16,%rsp # shouldn't be formally required
diff --git a/crypto/perlasm/arm-xlate.pl b/crypto/perlasm/arm-xlate.pl index e40065b..9a7f8c7 100755 --- a/crypto/perlasm/arm-xlate.pl +++ b/crypto/perlasm/arm-xlate.pl
@@ -215,7 +215,10 @@ if ($line =~ m/^\s*(#|@|\/\/)/) { print $line; next; } - $line =~ s|/\*.*\*/||; # get rid of C-style comments... + my @comments = (); + + $line =~ s|/\*(.*)\*/|| # get rid of C-style comments... + and push @comments, $1; $line =~ s|^\s+||; # ... and skip white spaces in beginning... $line =~ s|\s+$||; # ... and at the end @@ -232,6 +235,12 @@ } } + my $comments = join ' ', map { s|^\s+||; s|\s+$||; $_; } @comments; + my $commentprefix = @comments ? '// ' : ''; + my $commentspace = @comments ? ' ' : ''; + + my $pre_line = ''; + { $line =~ s|[\b\.]L(\w{2,})|L$1|g; # common denominator for Locallabel $line =~ s|\bL(\w{2,})|\.L$1|g if ($dotinlocallabels); @@ -258,10 +267,10 @@ # may add necessary relocations. It however is invalid to # mark the _first_ symbol of a section so, as it always is # considered an entry point. - printf ".alt_entry %s\n", $name; + $pre_line .= sprintf ".alt_entry %s\n", $name; } } - printf "%s:", $name; + $pre_line .= sprintf "%s:", $name; } } @@ -298,8 +307,11 @@ } } - print $line if ($line); - print "\n"; + $line = $pre_line . $line; + $commentspace = '' + if $line !~ /[^\n]$/; + + print $line, $commentspace, $commentprefix, $comments, "\n"; } print <<___;
diff --git a/crypto/perlasm/x86_64-xlate.pl b/crypto/perlasm/x86_64-xlate.pl index 86c83db..508d36f 100755 --- a/crypto/perlasm/x86_64-xlate.pl +++ b/crypto/perlasm/x86_64-xlate.pl
@@ -1322,9 +1322,9 @@ ######################################################################## +my $comment = "//"; +$comment = ";" if ($masm || $nasm); { - my $comment = "//"; - $comment = ";" if ($masm || $nasm); print <<___; $comment This file is generated from a similarly-named Perl script in the BoringSSL $comment source tree. Do not edit by hand. @@ -1374,24 +1374,36 @@ my $line = shift; $line =~ s|\R$||; # Better chomp + my @comments = (); + if ($nasm) { $line =~ s|^#ifdef |%ifdef |; $line =~ s|^#ifndef |%ifndef |; $line =~ s|^#endif|%endif|; - $line =~ s|[#!].*$||; # get rid of asm-style comments... + $line =~ s|[#!](.*)$|| # get rid of asm-style comments... + and push @comments, $1; } else { # Get rid of asm-style comments but not preprocessor directives. The # former are identified by having a letter after the '#' and starting in # the first column. $line =~ s|!.*$||; - $line =~ s|(?<=.)#.*$||; - $line =~ s|^#([^a-z].*)?$||; + $line =~ s|(?<=.)#(.*)$|| + and push @comments, $1; + $line =~ s|^#([^a-z].*)?$|| + and push @comments, $1; } - $line =~ s|/\*.*\*/||; # ... and C-style comments... + $line =~ s|/\*(.*)\*/|| # ... and C-style comments... + and push @comments, $1; $line =~ s|^\s+||; # ... and skip white spaces in beginning $line =~ s|\s+$||; # ... and at the end + my $comments = join ' ', map { s|^\s+||; s|\s+$||; $_; } @comments; + my $commentprefix = @comments ? "$comment " : ''; + my $commentspace = @comments ? ' ' : ''; + + my $pre_line = ''; + if (my $label=label->re(\$line)) { if ($gas) { my $name = ($globals{$label->{value}} or $label->{value}); @@ -1416,16 +1428,16 @@ } } } - print $label->out(); + $pre_line .= $label->out(); } if (my $directive=directive->re(\$line)) { - printf "%s",$directive->out(); + $pre_line .= $directive->out(); } elsif (my $opcode=opcode->re(\$line)) { my $asm = eval("\$".$opcode->mnemonic()); if ((ref($asm) eq 'CODE') && scalar(my @bytes=&$asm($line))) { - print $gas?".byte\t":"DB\t",join(',',@bytes),"\n"; + print $pre_line, $gas?".byte\t":"DB\t",join(',',@bytes),$commentspace,$commentprefix,$comment,"\n"; next; } @@ -1453,7 +1465,7 @@ if ($gas) { $insn = $opcode->out($#args>=1?$args[$#args]->size():$sz); @args = map($_->out($sz),@args); - printf "\t%s\t%s",$insn,join(",",@args); + $pre_line .= sprintf "\t%s\t%s",$insn,join(",",@args); } else { $insn = $opcode->out(); foreach (@args) { @@ -1466,14 +1478,18 @@ } @args = reverse(@args); undef $sz if ($nasm && $opcode->mnemonic() eq "lea"); - printf "\t%s\t%s",$insn,join(",",map($_->out($sz),@args)); + $pre_line .= sprintf "\t%s\t%s",$insn,join(",",map($_->out($sz),@args)); } } else { - printf "\t%s",$opcode->out(); + $pre_line .= sprintf "\t%s",$opcode->out(); } } - print $line,"\n"; + $line = $pre_line . $line; + $commentspace = '' + if $line !~ /[^\n]$/; + + print $line, $commentspace, $commentprefix, $comments, "\n"; } while(defined(my $line=<>)) {
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-apple.S b/gen/bcm/aes-gcm-avx2-x86_64-apple.S index cef98f8..e50dca2 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-apple.S +++ b/gen/bcm/aes-gcm-avx2-x86_64-apple.S
@@ -9,26 +9,26 @@ .p2align 4 aes_gcm_avx2_constants: - +// A shuffle mask that reflects the bytes of 16-byte blocks L$bswap_mask: .quad 0x08090a0b0c0d0e0f, 0x0001020304050607 - - - - - - - +// This is the GHASH reducing polynomial without its constant term, i.e. +// x^128 + x^7 + x^2 + x, represented using the backwards mapping +// between bits and polynomial coefficients. +// +// Alternatively, it can be interpreted as the naturally-ordered +// representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the +// "reversed" GHASH reducing polynomial without its x^128 term. L$gfpoly: .quad 1, 0xc200000000000000 - +// Same as above, but with the (1 << 64) bit set. L$gfpoly_and_internal_carrybit: .quad 1, 0xc200000000000001 .p2align 5 - +// The below constants are used for incrementing the counter blocks. L$ctr_pattern: .quad 0, 0 .quad 1, 0 @@ -48,14 +48,14 @@ - - +// Load the byte-reflected hash subkey. BoringSSL provides it in +// byte-reflected form except the two halves are in the wrong order. vpshufd $0x4e,(%rsi),%xmm3 - - - - +// Finish preprocessing the byte-reflected hash subkey by multiplying it by +// x^-1 ("standard" interpretation of polynomial coefficients) or +// equivalently x^1 (natural interpretation). This gets the key into a +// format that avoids having to bit-reflect the data blocks later. vpshufd $0xd3,%xmm3,%xmm0 vpsrad $31,%xmm0,%xmm0 vpaddq %xmm3,%xmm3,%xmm3 @@ -64,83 +64,83 @@ vbroadcasti128 L$gfpoly(%rip),%ymm6 - - vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 - vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 - vpshufd $0x4e,%xmm0,%xmm0 - vpxor %xmm0,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 - vpshufd $0x4e,%xmm1,%xmm1 - vpxor %xmm1,%xmm5,%xmm5 - vpxor %xmm0,%xmm5,%xmm5 +// Square H^1 to get H^2. + vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 // LO = a_L * a_L + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 // HI = a_H * a_H + vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm0,%xmm0 // Swap halves of LO + vpxor %xmm0,%xmm1,%xmm1 // Fold LO into MI + vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of MI + vpxor %xmm1,%xmm5,%xmm5 // Fold MI into HI (part 1) + vpxor %xmm0,%xmm5,%xmm5 // Fold MI into HI (part 2) - +// Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2]. vinserti128 $1,%xmm3,%ymm5,%ymm3 vinserti128 $1,%xmm5,%ymm5,%ymm5 - - vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 - vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 - vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpxor %ymm0,%ymm1,%ymm1 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 - vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm4,%ymm4 - vpxor %ymm0,%ymm4,%ymm4 +// Compute H_CUR2 = [H^4, H^3]. + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 // MI_1 = a_H * b_L + vpxor %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpxor %ymm0,%ymm1,%ymm1 // Fold LO into MI (part 1) + vpxor %ymm2,%ymm1,%ymm1 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpxor %ymm1,%ymm4,%ymm4 // Fold MI into HI (part 1) + vpxor %ymm0,%ymm4,%ymm4 // Fold MI into HI (part 2) - +// Store [H^2, H^1] and [H^4, H^3]. vmovdqu %ymm3,96(%rdi) vmovdqu %ymm4,64(%rdi) - - +// For Karatsuba multiplication: compute and store the two 64-bit halves of +// each key power XOR'd together. Order is 4,2,3,1. vpunpcklqdq %ymm3,%ymm4,%ymm0 vpunpckhqdq %ymm3,%ymm4,%ymm1 vpxor %ymm1,%ymm0,%ymm0 vmovdqu %ymm0,128+32(%rdi) +// Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7]. + vpclmulqdq $0x00,%ymm5,%ymm4,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm5,%ymm4,%ymm2 // MI_1 = a_H * b_L + vpxor %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpxor %ymm0,%ymm1,%ymm1 // Fold LO into MI (part 1) + vpxor %ymm2,%ymm1,%ymm1 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm5,%ymm4,%ymm3 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpxor %ymm1,%ymm3,%ymm3 // Fold MI into HI (part 1) + vpxor %ymm0,%ymm3,%ymm3 // Fold MI into HI (part 2) - vpclmulqdq $0x00,%ymm5,%ymm4,%ymm0 - vpclmulqdq $0x01,%ymm5,%ymm4,%ymm1 - vpclmulqdq $0x10,%ymm5,%ymm4,%ymm2 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpxor %ymm0,%ymm1,%ymm1 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x11,%ymm5,%ymm4,%ymm3 - vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm3,%ymm3 - vpxor %ymm0,%ymm3,%ymm3 - - vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 - vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 - vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpxor %ymm0,%ymm1,%ymm1 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 - vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm4,%ymm4 - vpxor %ymm0,%ymm4,%ymm4 + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 // MI_1 = a_H * b_L + vpxor %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpxor %ymm0,%ymm1,%ymm1 // Fold LO into MI (part 1) + vpxor %ymm2,%ymm1,%ymm1 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpxor %ymm1,%ymm4,%ymm4 // Fold MI into HI (part 1) + vpxor %ymm0,%ymm4,%ymm4 // Fold MI into HI (part 2) vmovdqu %ymm3,32(%rdi) vmovdqu %ymm4,0(%rdi) - - +// Again, compute and store the two 64-bit halves of each key power XOR'd +// together. Order is 8,6,7,5. vpunpcklqdq %ymm3,%ymm4,%ymm0 vpunpckhqdq %ymm3,%ymm4,%ymm1 vpxor %ymm1,%ymm0,%ymm0 @@ -169,25 +169,25 @@ vmovdqu L$gfpoly(%rip),%xmm3 vpshufb %xmm1,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 - vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 - vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 - vpxor %xmm6,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 - vpshufd $0x4e,%xmm4,%xmm4 - vpxor %xmm4,%xmm5,%xmm5 - vpxor %xmm6,%xmm5,%xmm5 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 - vpshufd $0x4e,%xmm5,%xmm5 - vpxor %xmm5,%xmm0,%xmm0 - vpxor %xmm4,%xmm0,%xmm0 + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 // MI_1 = a_H * b_L + vpxor %xmm6,%xmm5,%xmm5 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm4,%xmm4 // Swap halves of LO + vpxor %xmm4,%xmm5,%xmm5 // Fold LO into MI (part 1) + vpxor %xmm6,%xmm5,%xmm5 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm5,%xmm5 // Swap halves of MI + vpxor %xmm5,%xmm0,%xmm0 // Fold MI into HI (part 1) + vpxor %xmm4,%xmm0,%xmm0 // Fold MI into HI (part 2) vpshufb %xmm1,%xmm0,%xmm0 vmovdqu %xmm0,(%rdi) - +// No need for vzeroupper, since only xmm registers were used. ret @@ -204,33 +204,33 @@ - - - +// Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, +// usually only 128-bit vectors will be used. So as an optimization, don't +// broadcast these constants to both 128-bit lanes quite yet. vmovdqu L$bswap_mask(%rip),%xmm6 vmovdqu L$gfpoly(%rip),%xmm7 - +// Load the GHASH accumulator. vmovdqu (%rdi),%xmm5 vpshufb %xmm6,%xmm5,%xmm5 - +// Optimize for AADLEN < 32 by checking for AADLEN < 32 before AADLEN < 128. cmpq $32,%rcx jb L$ghash_lastblock - - +// AADLEN >= 32, so we'll operate on full vectors. Broadcast bswap_mask and +// gfpoly to both 128-bit lanes. vinserti128 $1,%xmm6,%ymm6,%ymm6 vinserti128 $1,%xmm7,%ymm7,%ymm7 cmpq $127,%rcx jbe L$ghash_loop_1x - +// Update GHASH with 128 bytes of AAD at a time. vmovdqu 128(%rsi),%ymm8 vmovdqu 128+32(%rsi),%ymm9 L$ghash_loop_4x: - +// First vector vmovdqu 0(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 0(%rsi),%ymm2 @@ -240,7 +240,7 @@ vpunpckhqdq %ymm1,%ymm1,%ymm0 vpxor %ymm1,%ymm0,%ymm0 vpclmulqdq $0x00,%ymm8,%ymm0,%ymm4 - +// Second vector vmovdqu 32(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 32(%rsi),%ymm2 @@ -252,7 +252,7 @@ vpxor %ymm1,%ymm0,%ymm0 vpclmulqdq $0x10,%ymm8,%ymm0,%ymm0 vpxor %ymm0,%ymm4,%ymm4 - +// Third vector vmovdqu 64(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 64(%rsi),%ymm2 @@ -265,7 +265,7 @@ vpclmulqdq $0x00,%ymm9,%ymm0,%ymm0 vpxor %ymm0,%ymm4,%ymm4 - +// Fourth vector vmovdqu 96(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 96(%rsi),%ymm2 @@ -277,17 +277,17 @@ vpxor %ymm1,%ymm0,%ymm0 vpclmulqdq $0x10,%ymm9,%ymm0,%ymm0 vpxor %ymm0,%ymm4,%ymm4 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm3,%ymm4,%ymm4 vpxor %ymm5,%ymm4,%ymm4 - +// Fold lo into mi. vbroadcasti128 L$gfpoly(%rip),%ymm2 vpclmulqdq $0x01,%ymm3,%ymm2,%ymm0 vpshufd $0x4e,%ymm3,%ymm3 vpxor %ymm3,%ymm4,%ymm4 vpxor %ymm0,%ymm4,%ymm4 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm4,%ymm2,%ymm0 vpshufd $0x4e,%ymm4,%ymm4 vpxor %ymm4,%ymm5,%ymm5 @@ -295,12 +295,12 @@ vextracti128 $1,%ymm5,%xmm0 vpxor %xmm0,%xmm5,%xmm5 - subq $-128,%rdx + subq $-128,%rdx // 128 is 4 bytes, -128 is 1 byte addq $-128,%rcx cmpq $127,%rcx ja L$ghash_loop_4x - +// Update GHASH with 32 bytes of AAD at a time. cmpq $32,%rcx jb L$ghash_loop_1x_done L$ghash_loop_1x: @@ -308,19 +308,19 @@ vpshufb %ymm6,%ymm0,%ymm0 vpxor %ymm0,%ymm5,%ymm5 vmovdqu 128-32(%rsi),%ymm0 - vpclmulqdq $0x00,%ymm0,%ymm5,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 - vpclmulqdq $0x10,%ymm0,%ymm5,%ymm3 - vpxor %ymm3,%ymm2,%ymm2 - vpclmulqdq $0x01,%ymm1,%ymm7,%ymm3 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm2,%ymm2 - vpxor %ymm3,%ymm2,%ymm2 - vpclmulqdq $0x11,%ymm0,%ymm5,%ymm5 - vpclmulqdq $0x01,%ymm2,%ymm7,%ymm1 - vpshufd $0x4e,%ymm2,%ymm2 - vpxor %ymm2,%ymm5,%ymm5 - vpxor %ymm1,%ymm5,%ymm5 + vpclmulqdq $0x00,%ymm0,%ymm5,%ymm1 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm0,%ymm5,%ymm3 // MI_1 = a_H * b_L + vpxor %ymm3,%ymm2,%ymm2 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm1,%ymm7,%ymm3 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of LO + vpxor %ymm1,%ymm2,%ymm2 // Fold LO into MI (part 1) + vpxor %ymm3,%ymm2,%ymm2 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm0,%ymm5,%ymm5 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm2,%ymm7,%ymm1 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm2,%ymm2 // Swap halves of MI + vpxor %ymm2,%ymm5,%ymm5 // Fold MI into HI (part 1) + vpxor %ymm1,%ymm5,%ymm5 // Fold MI into HI (part 2) vextracti128 $1,%ymm5,%xmm0 vpxor %xmm0,%xmm5,%xmm5 @@ -330,7 +330,7 @@ jae L$ghash_loop_1x L$ghash_loop_1x_done: - +// Update GHASH with the remaining 16-byte block if any. L$ghash_lastblock: testq %rcx,%rcx jz L$ghash_done @@ -338,23 +338,23 @@ vpshufb %xmm6,%xmm0,%xmm0 vpxor %xmm0,%xmm5,%xmm5 vmovdqu 128-16(%rsi),%xmm0 - vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 - vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3 - vpxor %xmm3,%xmm2,%xmm2 - vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3 - vpshufd $0x4e,%xmm1,%xmm1 - vpxor %xmm1,%xmm2,%xmm2 - vpxor %xmm3,%xmm2,%xmm2 - vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1 - vpshufd $0x4e,%xmm2,%xmm2 - vpxor %xmm2,%xmm5,%xmm5 - vpxor %xmm1,%xmm5,%xmm5 + vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3 // MI_1 = a_H * b_L + vpxor %xmm3,%xmm2,%xmm2 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of LO + vpxor %xmm1,%xmm2,%xmm2 // Fold LO into MI (part 1) + vpxor %xmm3,%xmm2,%xmm2 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm2,%xmm2 // Swap halves of MI + vpxor %xmm2,%xmm5,%xmm5 // Fold MI into HI (part 1) + vpxor %xmm1,%xmm5,%xmm5 // Fold MI into HI (part 2) L$ghash_done: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm6,%xmm5,%xmm5 vmovdqu %xmm5,(%rdi) @@ -375,45 +375,45 @@ pushq %r12 - movq 16(%rsp),%r12 + movq 16(%rsp),%r12 // arg7 #ifdef BORINGSSL_DISPATCH_TEST movb $1,_BORINGSSL_function_hit+6(%rip) #endif vbroadcasti128 L$bswap_mask(%rip),%ymm0 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm1 vpshufb %xmm0,%xmm1,%xmm1 vbroadcasti128 (%r8),%ymm11 vpshufb %ymm0,%ymm11,%ymm11 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti128 (%rcx),%ymm9 vbroadcasti128 (%r11),%ymm10 - +// Finish initializing LE_CTR by adding 1 to the second block. vpaddd L$ctr_pattern(%rip),%ymm11,%ymm11 - - +// If there are at least 128 bytes of data, then continue into the loop that +// processes 128 bytes of data at a time. Otherwise skip it. cmpq $127,%rdx jbe L$crypt_loop_4x_done__func1 vmovdqu 128(%r9),%ymm7 vmovdqu 128+32(%r9),%ymm8 - - - +// Encrypt the first 4 vectors of plaintext blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu L$inc_2blocks(%rip),%ymm2 vpshufb %ymm0,%ymm11,%ymm12 vpaddd %ymm2,%ymm11,%ymm11 @@ -424,7 +424,7 @@ vpshufb %ymm0,%ymm11,%ymm15 vpaddd %ymm2,%ymm11,%ymm11 - +// AES "round zero": XOR in the zero-th round key. vpxor %ymm9,%ymm12,%ymm12 vpxor %ymm9,%ymm13,%ymm13 vpxor %ymm9,%ymm14,%ymm14 @@ -454,16 +454,16 @@ vmovdqu %ymm14,64(%rsi) vmovdqu %ymm15,96(%rsi) - subq $-128,%rdi + subq $-128,%rdi // 128 is 4 bytes, -128 is 1 byte addq $-128,%rdx cmpq $127,%rdx jbe L$ghash_last_ciphertext_4x__func1 .p2align 4 L$crypt_loop_4x__func1: - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu L$inc_2blocks(%rip),%ymm2 vpshufb %ymm0,%ymm11,%ymm12 vpaddd %ymm2,%ymm11,%ymm11 @@ -474,7 +474,7 @@ vpshufb %ymm0,%ymm11,%ymm15 vpaddd %ymm2,%ymm11,%ymm11 - +// AES "round zero": XOR in the zero-th round key. vpxor %ymm9,%ymm12,%ymm12 vpxor %ymm9,%ymm13,%ymm13 vpxor %ymm9,%ymm14,%ymm14 @@ -483,7 +483,7 @@ cmpl $24,%r10d jl L$aes128__func1 je L$aes192__func1 - +// AES-256 vbroadcasti128 -208(%r11),%ymm2 vaesenc %ymm2,%ymm12,%ymm12 vaesenc %ymm2,%ymm13,%ymm13 @@ -512,7 +512,7 @@ L$aes128__func1: prefetcht0 512(%rdi) prefetcht0 512+64(%rdi) - +// First vector vmovdqu 0(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 0(%r9),%ymm4 @@ -536,7 +536,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Second vector vmovdqu 32(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 32(%r9),%ymm4 @@ -555,7 +555,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Third vector vmovdqu 64(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 64(%r9),%ymm4 @@ -582,7 +582,7 @@ vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Fourth vector vmovdqu 96(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 @@ -608,11 +608,11 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm1,%ymm6,%ymm6 - +// Fold lo into mi. vbroadcasti128 L$gfpoly(%rip),%ymm4 vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 vpshufd $0x4e,%ymm5,%ymm5 @@ -625,7 +625,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 vpshufd $0x4e,%ymm6,%ymm6 vpxor %ymm6,%ymm1,%ymm1 @@ -641,7 +641,7 @@ vpxor %xmm2,%xmm1,%xmm1 - subq $-128,%rsi + subq $-128,%rsi // 128 is 4 bytes, -128 is 1 byte vpxor 0(%rdi),%ymm10,%ymm2 vpxor 32(%rdi),%ymm10,%ymm3 vpxor 64(%rdi),%ymm10,%ymm5 @@ -661,7 +661,7 @@ cmpq $127,%rdx ja L$crypt_loop_4x__func1 L$ghash_last_ciphertext_4x__func1: - +// First vector vmovdqu 0(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 0(%r9),%ymm4 @@ -671,7 +671,7 @@ vpunpckhqdq %ymm3,%ymm3,%ymm2 vpxor %ymm3,%ymm2,%ymm2 vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 - +// Second vector vmovdqu 32(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 32(%r9),%ymm4 @@ -683,7 +683,7 @@ vpxor %ymm3,%ymm2,%ymm2 vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Third vector vmovdqu 64(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 64(%r9),%ymm4 @@ -696,7 +696,7 @@ vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Fourth vector vmovdqu 96(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 96(%r9),%ymm4 @@ -708,17 +708,17 @@ vpxor %ymm3,%ymm2,%ymm2 vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm1,%ymm6,%ymm6 - +// Fold lo into mi. vbroadcasti128 L$gfpoly(%rip),%ymm4 vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 vpshufd $0x4e,%ymm5,%ymm5 vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm2,%ymm6,%ymm6 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 vpshufd $0x4e,%ymm6,%ymm6 vpxor %ymm6,%ymm1,%ymm1 @@ -728,18 +728,18 @@ subq $-128,%rsi L$crypt_loop_4x_done__func1: - +// Check whether any data remains. testq %rdx,%rdx jz L$done__func1 +// DATALEN is in [16, 32, 48, 64, 80, 96, 112]. - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. leaq 128(%r9),%r8 subq %rdx,%r8 - +// Start collecting the unreduced GHASH intermediate value LO, MI, HI. vpxor %xmm5,%xmm5,%xmm5 vpxor %xmm6,%xmm6,%xmm6 vpxor %xmm7,%xmm7,%xmm7 @@ -747,7 +747,7 @@ cmpq $64,%rdx jb L$lessthan64bytes__func1 - +// DATALEN is in [64, 80, 96, 112]. Encrypt two vectors of counter blocks. vpshufb %ymm0,%ymm11,%ymm12 vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11 vpshufb %ymm0,%ymm11,%ymm13 @@ -765,7 +765,7 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - +// XOR the data with the two vectors of keystream blocks. vmovdqu 0(%rdi),%ymm2 vmovdqu 32(%rdi),%ymm3 vpxor %ymm2,%ymm12,%ymm12 @@ -773,7 +773,7 @@ vmovdqu %ymm12,0(%rsi) vmovdqu %ymm13,32(%rsi) - +// Update GHASH with two vectors of ciphertext blocks, without reducing. vpshufb %ymm0,%ymm12,%ymm12 vpshufb %ymm0,%ymm13,%ymm13 vpxor %ymm1,%ymm12,%ymm12 @@ -801,7 +801,7 @@ vpxor %xmm1,%xmm1,%xmm1 - +// DATALEN is in [16, 32, 48]. Encrypt two last vectors of counter blocks. L$lessthan64bytes__func1: vpshufb %ymm0,%ymm11,%ymm12 vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11 @@ -819,8 +819,8 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - - +// XOR the remaining data with the keystream blocks, and update GHASH with +// the remaining ciphertext blocks without reducing. cmpq $32,%rdx jb L$xor_one_block__func1 @@ -877,7 +877,7 @@ vpxor %ymm4,%ymm7,%ymm7 L$reduce__func1: - +// Finally, do the GHASH reduction. vbroadcasti128 L$gfpoly(%rip),%ymm2 vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 vpshufd $0x4e,%ymm5,%ymm5 @@ -891,7 +891,7 @@ vpxor %xmm7,%xmm1,%xmm1 L$done__func1: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm0,%xmm1,%xmm1 vmovdqu %xmm1,(%r12) @@ -914,33 +914,33 @@ pushq %r12 - movq 16(%rsp),%r12 + movq 16(%rsp),%r12 // arg7 vbroadcasti128 L$bswap_mask(%rip),%ymm0 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm1 vpshufb %xmm0,%xmm1,%xmm1 vbroadcasti128 (%r8),%ymm11 vpshufb %ymm0,%ymm11,%ymm11 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti128 (%rcx),%ymm9 vbroadcasti128 (%r11),%ymm10 - +// Finish initializing LE_CTR by adding 1 to the second block. vpaddd L$ctr_pattern(%rip),%ymm11,%ymm11 - - +// If there are at least 128 bytes of data, then continue into the loop that +// processes 128 bytes of data at a time. Otherwise skip it. cmpq $127,%rdx jbe L$crypt_loop_4x_done__func2 @@ -949,9 +949,9 @@ .p2align 4 L$crypt_loop_4x__func2: - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu L$inc_2blocks(%rip),%ymm2 vpshufb %ymm0,%ymm11,%ymm12 vpaddd %ymm2,%ymm11,%ymm11 @@ -962,7 +962,7 @@ vpshufb %ymm0,%ymm11,%ymm15 vpaddd %ymm2,%ymm11,%ymm11 - +// AES "round zero": XOR in the zero-th round key. vpxor %ymm9,%ymm12,%ymm12 vpxor %ymm9,%ymm13,%ymm13 vpxor %ymm9,%ymm14,%ymm14 @@ -971,7 +971,7 @@ cmpl $24,%r10d jl L$aes128__func2 je L$aes192__func2 - +// AES-256 vbroadcasti128 -208(%r11),%ymm2 vaesenc %ymm2,%ymm12,%ymm12 vaesenc %ymm2,%ymm13,%ymm13 @@ -1000,7 +1000,7 @@ L$aes128__func2: prefetcht0 512(%rdi) prefetcht0 512+64(%rdi) - +// First vector vmovdqu 0(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 0(%r9),%ymm4 @@ -1024,7 +1024,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Second vector vmovdqu 32(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 32(%r9),%ymm4 @@ -1043,7 +1043,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Third vector vmovdqu 64(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 64(%r9),%ymm4 @@ -1070,7 +1070,7 @@ vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Fourth vector vmovdqu 96(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 @@ -1096,11 +1096,11 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm1,%ymm6,%ymm6 - +// Fold lo into mi. vbroadcasti128 L$gfpoly(%rip),%ymm4 vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 vpshufd $0x4e,%ymm5,%ymm5 @@ -1113,7 +1113,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 vpshufd $0x4e,%ymm6,%ymm6 vpxor %ymm6,%ymm1,%ymm1 @@ -1129,7 +1129,7 @@ vpxor %xmm2,%xmm1,%xmm1 - +// 128 is 4 bytes, -128 is 1 byte vpxor 0(%rdi),%ymm10,%ymm2 vpxor 32(%rdi),%ymm10,%ymm3 vpxor 64(%rdi),%ymm10,%ymm5 @@ -1149,18 +1149,18 @@ cmpq $127,%rdx ja L$crypt_loop_4x__func2 L$crypt_loop_4x_done__func2: - +// Check whether any data remains. testq %rdx,%rdx jz L$done__func2 +// DATALEN is in [16, 32, 48, 64, 80, 96, 112]. - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. leaq 128(%r9),%r8 subq %rdx,%r8 - +// Start collecting the unreduced GHASH intermediate value LO, MI, HI. vpxor %xmm5,%xmm5,%xmm5 vpxor %xmm6,%xmm6,%xmm6 vpxor %xmm7,%xmm7,%xmm7 @@ -1168,7 +1168,7 @@ cmpq $64,%rdx jb L$lessthan64bytes__func2 - +// DATALEN is in [64, 80, 96, 112]. Encrypt two vectors of counter blocks. vpshufb %ymm0,%ymm11,%ymm12 vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11 vpshufb %ymm0,%ymm11,%ymm13 @@ -1186,7 +1186,7 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - +// XOR the data with the two vectors of keystream blocks. vmovdqu 0(%rdi),%ymm2 vmovdqu 32(%rdi),%ymm3 vpxor %ymm2,%ymm12,%ymm12 @@ -1194,7 +1194,7 @@ vmovdqu %ymm12,0(%rsi) vmovdqu %ymm13,32(%rsi) - +// Update GHASH with two vectors of ciphertext blocks, without reducing. vpshufb %ymm0,%ymm2,%ymm12 vpshufb %ymm0,%ymm3,%ymm13 vpxor %ymm1,%ymm12,%ymm12 @@ -1222,7 +1222,7 @@ vpxor %xmm1,%xmm1,%xmm1 - +// DATALEN is in [16, 32, 48]. Encrypt two last vectors of counter blocks. L$lessthan64bytes__func2: vpshufb %ymm0,%ymm11,%ymm12 vpaddd L$inc_2blocks(%rip),%ymm11,%ymm11 @@ -1240,8 +1240,8 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - - +// XOR the remaining data with the keystream blocks, and update GHASH with +// the remaining ciphertext blocks without reducing. cmpq $32,%rdx jb L$xor_one_block__func2 @@ -1298,7 +1298,7 @@ vpxor %ymm4,%ymm7,%ymm7 L$reduce__func2: - +// Finally, do the GHASH reduction. vbroadcasti128 L$gfpoly(%rip),%ymm2 vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 vpshufd $0x4e,%ymm5,%ymm5 @@ -1312,7 +1312,7 @@ vpxor %xmm7,%xmm1,%xmm1 L$done__func2: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm0,%xmm1,%xmm1 vmovdqu %xmm1,(%r12)
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-linux.S b/gen/bcm/aes-gcm-avx2-x86_64-linux.S index fa47e2e..e21f712 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-linux.S +++ b/gen/bcm/aes-gcm-avx2-x86_64-linux.S
@@ -9,26 +9,26 @@ .align 16 aes_gcm_avx2_constants: - +// A shuffle mask that reflects the bytes of 16-byte blocks .Lbswap_mask: .quad 0x08090a0b0c0d0e0f, 0x0001020304050607 - - - - - - - +// This is the GHASH reducing polynomial without its constant term, i.e. +// x^128 + x^7 + x^2 + x, represented using the backwards mapping +// between bits and polynomial coefficients. +// +// Alternatively, it can be interpreted as the naturally-ordered +// representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the +// "reversed" GHASH reducing polynomial without its x^128 term. .Lgfpoly: .quad 1, 0xc200000000000000 - +// Same as above, but with the (1 << 64) bit set. .Lgfpoly_and_internal_carrybit: .quad 1, 0xc200000000000001 .align 32 - +// The below constants are used for incrementing the counter blocks. .Lctr_pattern: .quad 0, 0 .quad 1, 0 @@ -48,14 +48,14 @@ - - +// Load the byte-reflected hash subkey. BoringSSL provides it in +// byte-reflected form except the two halves are in the wrong order. vpshufd $0x4e,(%rsi),%xmm3 - - - - +// Finish preprocessing the byte-reflected hash subkey by multiplying it by +// x^-1 ("standard" interpretation of polynomial coefficients) or +// equivalently x^1 (natural interpretation). This gets the key into a +// format that avoids having to bit-reflect the data blocks later. vpshufd $0xd3,%xmm3,%xmm0 vpsrad $31,%xmm0,%xmm0 vpaddq %xmm3,%xmm3,%xmm3 @@ -64,83 +64,83 @@ vbroadcasti128 .Lgfpoly(%rip),%ymm6 - - vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 - vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 - vpshufd $0x4e,%xmm0,%xmm0 - vpxor %xmm0,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 - vpshufd $0x4e,%xmm1,%xmm1 - vpxor %xmm1,%xmm5,%xmm5 - vpxor %xmm0,%xmm5,%xmm5 +// Square H^1 to get H^2. + vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 // LO = a_L * a_L + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm5 // HI = a_H * a_H + vpclmulqdq $0x01,%xmm0,%xmm6,%xmm1 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm0,%xmm0 // Swap halves of LO + vpxor %xmm0,%xmm1,%xmm1 // Fold LO into MI + vpclmulqdq $0x01,%xmm1,%xmm6,%xmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of MI + vpxor %xmm1,%xmm5,%xmm5 // Fold MI into HI (part 1) + vpxor %xmm0,%xmm5,%xmm5 // Fold MI into HI (part 2) - +// Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2]. vinserti128 $1,%xmm3,%ymm5,%ymm3 vinserti128 $1,%xmm5,%ymm5,%ymm5 - - vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 - vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 - vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpxor %ymm0,%ymm1,%ymm1 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 - vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm4,%ymm4 - vpxor %ymm0,%ymm4,%ymm4 +// Compute H_CUR2 = [H^4, H^3]. + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 // MI_1 = a_H * b_L + vpxor %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpxor %ymm0,%ymm1,%ymm1 // Fold LO into MI (part 1) + vpxor %ymm2,%ymm1,%ymm1 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpxor %ymm1,%ymm4,%ymm4 // Fold MI into HI (part 1) + vpxor %ymm0,%ymm4,%ymm4 // Fold MI into HI (part 2) - +// Store [H^2, H^1] and [H^4, H^3]. vmovdqu %ymm3,96(%rdi) vmovdqu %ymm4,64(%rdi) - - +// For Karatsuba multiplication: compute and store the two 64-bit halves of +// each key power XOR'd together. Order is 4,2,3,1. vpunpcklqdq %ymm3,%ymm4,%ymm0 vpunpckhqdq %ymm3,%ymm4,%ymm1 vpxor %ymm1,%ymm0,%ymm0 vmovdqu %ymm0,128+32(%rdi) +// Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7]. + vpclmulqdq $0x00,%ymm5,%ymm4,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm5,%ymm4,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm5,%ymm4,%ymm2 // MI_1 = a_H * b_L + vpxor %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpxor %ymm0,%ymm1,%ymm1 // Fold LO into MI (part 1) + vpxor %ymm2,%ymm1,%ymm1 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm5,%ymm4,%ymm3 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpxor %ymm1,%ymm3,%ymm3 // Fold MI into HI (part 1) + vpxor %ymm0,%ymm3,%ymm3 // Fold MI into HI (part 2) - vpclmulqdq $0x00,%ymm5,%ymm4,%ymm0 - vpclmulqdq $0x01,%ymm5,%ymm4,%ymm1 - vpclmulqdq $0x10,%ymm5,%ymm4,%ymm2 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpxor %ymm0,%ymm1,%ymm1 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x11,%ymm5,%ymm4,%ymm3 - vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm3,%ymm3 - vpxor %ymm0,%ymm3,%ymm3 - - vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 - vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 - vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpxor %ymm0,%ymm1,%ymm1 - vpxor %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 - vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm4,%ymm4 - vpxor %ymm0,%ymm4,%ymm4 + vpclmulqdq $0x00,%ymm5,%ymm3,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm5,%ymm3,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm5,%ymm3,%ymm2 // MI_1 = a_H * b_L + vpxor %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm6,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpxor %ymm0,%ymm1,%ymm1 // Fold LO into MI (part 1) + vpxor %ymm2,%ymm1,%ymm1 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm5,%ymm3,%ymm4 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm6,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpxor %ymm1,%ymm4,%ymm4 // Fold MI into HI (part 1) + vpxor %ymm0,%ymm4,%ymm4 // Fold MI into HI (part 2) vmovdqu %ymm3,32(%rdi) vmovdqu %ymm4,0(%rdi) - - +// Again, compute and store the two 64-bit halves of each key power XOR'd +// together. Order is 8,6,7,5. vpunpcklqdq %ymm3,%ymm4,%ymm0 vpunpckhqdq %ymm3,%ymm4,%ymm1 vpxor %ymm1,%ymm0,%ymm0 @@ -168,25 +168,25 @@ vmovdqu .Lgfpoly(%rip),%xmm3 vpshufb %xmm1,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 - vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 - vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 - vpxor %xmm6,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 - vpshufd $0x4e,%xmm4,%xmm4 - vpxor %xmm4,%xmm5,%xmm5 - vpxor %xmm6,%xmm5,%xmm5 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 - vpshufd $0x4e,%xmm5,%xmm5 - vpxor %xmm5,%xmm0,%xmm0 - vpxor %xmm4,%xmm0,%xmm0 + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 // MI_1 = a_H * b_L + vpxor %xmm6,%xmm5,%xmm5 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm4,%xmm4 // Swap halves of LO + vpxor %xmm4,%xmm5,%xmm5 // Fold LO into MI (part 1) + vpxor %xmm6,%xmm5,%xmm5 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm5,%xmm5 // Swap halves of MI + vpxor %xmm5,%xmm0,%xmm0 // Fold MI into HI (part 1) + vpxor %xmm4,%xmm0,%xmm0 // Fold MI into HI (part 2) vpshufb %xmm1,%xmm0,%xmm0 vmovdqu %xmm0,(%rdi) - +// No need for vzeroupper, since only xmm registers were used. ret .cfi_endproc @@ -202,33 +202,33 @@ - - - +// Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, +// usually only 128-bit vectors will be used. So as an optimization, don't +// broadcast these constants to both 128-bit lanes quite yet. vmovdqu .Lbswap_mask(%rip),%xmm6 vmovdqu .Lgfpoly(%rip),%xmm7 - +// Load the GHASH accumulator. vmovdqu (%rdi),%xmm5 vpshufb %xmm6,%xmm5,%xmm5 - +// Optimize for AADLEN < 32 by checking for AADLEN < 32 before AADLEN < 128. cmpq $32,%rcx jb .Lghash_lastblock - - +// AADLEN >= 32, so we'll operate on full vectors. Broadcast bswap_mask and +// gfpoly to both 128-bit lanes. vinserti128 $1,%xmm6,%ymm6,%ymm6 vinserti128 $1,%xmm7,%ymm7,%ymm7 cmpq $127,%rcx jbe .Lghash_loop_1x - +// Update GHASH with 128 bytes of AAD at a time. vmovdqu 128(%rsi),%ymm8 vmovdqu 128+32(%rsi),%ymm9 .Lghash_loop_4x: - +// First vector vmovdqu 0(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 0(%rsi),%ymm2 @@ -238,7 +238,7 @@ vpunpckhqdq %ymm1,%ymm1,%ymm0 vpxor %ymm1,%ymm0,%ymm0 vpclmulqdq $0x00,%ymm8,%ymm0,%ymm4 - +// Second vector vmovdqu 32(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 32(%rsi),%ymm2 @@ -250,7 +250,7 @@ vpxor %ymm1,%ymm0,%ymm0 vpclmulqdq $0x10,%ymm8,%ymm0,%ymm0 vpxor %ymm0,%ymm4,%ymm4 - +// Third vector vmovdqu 64(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 64(%rsi),%ymm2 @@ -263,7 +263,7 @@ vpclmulqdq $0x00,%ymm9,%ymm0,%ymm0 vpxor %ymm0,%ymm4,%ymm4 - +// Fourth vector vmovdqu 96(%rdx),%ymm1 vpshufb %ymm6,%ymm1,%ymm1 vmovdqu 96(%rsi),%ymm2 @@ -275,17 +275,17 @@ vpxor %ymm1,%ymm0,%ymm0 vpclmulqdq $0x10,%ymm9,%ymm0,%ymm0 vpxor %ymm0,%ymm4,%ymm4 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm3,%ymm4,%ymm4 vpxor %ymm5,%ymm4,%ymm4 - +// Fold lo into mi. vbroadcasti128 .Lgfpoly(%rip),%ymm2 vpclmulqdq $0x01,%ymm3,%ymm2,%ymm0 vpshufd $0x4e,%ymm3,%ymm3 vpxor %ymm3,%ymm4,%ymm4 vpxor %ymm0,%ymm4,%ymm4 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm4,%ymm2,%ymm0 vpshufd $0x4e,%ymm4,%ymm4 vpxor %ymm4,%ymm5,%ymm5 @@ -293,12 +293,12 @@ vextracti128 $1,%ymm5,%xmm0 vpxor %xmm0,%xmm5,%xmm5 - subq $-128,%rdx + subq $-128,%rdx // 128 is 4 bytes, -128 is 1 byte addq $-128,%rcx cmpq $127,%rcx ja .Lghash_loop_4x - +// Update GHASH with 32 bytes of AAD at a time. cmpq $32,%rcx jb .Lghash_loop_1x_done .Lghash_loop_1x: @@ -306,19 +306,19 @@ vpshufb %ymm6,%ymm0,%ymm0 vpxor %ymm0,%ymm5,%ymm5 vmovdqu 128-32(%rsi),%ymm0 - vpclmulqdq $0x00,%ymm0,%ymm5,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 - vpclmulqdq $0x10,%ymm0,%ymm5,%ymm3 - vpxor %ymm3,%ymm2,%ymm2 - vpclmulqdq $0x01,%ymm1,%ymm7,%ymm3 - vpshufd $0x4e,%ymm1,%ymm1 - vpxor %ymm1,%ymm2,%ymm2 - vpxor %ymm3,%ymm2,%ymm2 - vpclmulqdq $0x11,%ymm0,%ymm5,%ymm5 - vpclmulqdq $0x01,%ymm2,%ymm7,%ymm1 - vpshufd $0x4e,%ymm2,%ymm2 - vpxor %ymm2,%ymm5,%ymm5 - vpxor %ymm1,%ymm5,%ymm5 + vpclmulqdq $0x00,%ymm0,%ymm5,%ymm1 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm0,%ymm5,%ymm3 // MI_1 = a_H * b_L + vpxor %ymm3,%ymm2,%ymm2 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm1,%ymm7,%ymm3 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of LO + vpxor %ymm1,%ymm2,%ymm2 // Fold LO into MI (part 1) + vpxor %ymm3,%ymm2,%ymm2 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%ymm0,%ymm5,%ymm5 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm2,%ymm7,%ymm1 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm2,%ymm2 // Swap halves of MI + vpxor %ymm2,%ymm5,%ymm5 // Fold MI into HI (part 1) + vpxor %ymm1,%ymm5,%ymm5 // Fold MI into HI (part 2) vextracti128 $1,%ymm5,%xmm0 vpxor %xmm0,%xmm5,%xmm5 @@ -328,7 +328,7 @@ jae .Lghash_loop_1x .Lghash_loop_1x_done: - +// Update GHASH with the remaining 16-byte block if any. .Lghash_lastblock: testq %rcx,%rcx jz .Lghash_done @@ -336,23 +336,23 @@ vpshufb %xmm6,%xmm0,%xmm0 vpxor %xmm0,%xmm5,%xmm5 vmovdqu 128-16(%rsi),%xmm0 - vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 - vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3 - vpxor %xmm3,%xmm2,%xmm2 - vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3 - vpshufd $0x4e,%xmm1,%xmm1 - vpxor %xmm1,%xmm2,%xmm2 - vpxor %xmm3,%xmm2,%xmm2 - vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1 - vpshufd $0x4e,%xmm2,%xmm2 - vpxor %xmm2,%xmm5,%xmm5 - vpxor %xmm1,%xmm5,%xmm5 + vpclmulqdq $0x00,%xmm0,%xmm5,%xmm1 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm2 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm0,%xmm5,%xmm3 // MI_1 = a_H * b_L + vpxor %xmm3,%xmm2,%xmm2 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm1,%xmm7,%xmm3 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of LO + vpxor %xmm1,%xmm2,%xmm2 // Fold LO into MI (part 1) + vpxor %xmm3,%xmm2,%xmm2 // Fold LO into MI (part 2) + vpclmulqdq $0x11,%xmm0,%xmm5,%xmm5 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm2,%xmm7,%xmm1 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm2,%xmm2 // Swap halves of MI + vpxor %xmm2,%xmm5,%xmm5 // Fold MI into HI (part 1) + vpxor %xmm1,%xmm5,%xmm5 // Fold MI into HI (part 2) .Lghash_done: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm6,%xmm5,%xmm5 vmovdqu %xmm5,(%rdi) @@ -373,7 +373,7 @@ .cfi_adjust_cfa_offset 8 .cfi_offset %r12,-16 - movq 16(%rsp),%r12 + movq 16(%rsp),%r12 // arg7 #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit .hidden BORINGSSL_function_hit @@ -381,38 +381,38 @@ #endif vbroadcasti128 .Lbswap_mask(%rip),%ymm0 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm1 vpshufb %xmm0,%xmm1,%xmm1 vbroadcasti128 (%r8),%ymm11 vpshufb %ymm0,%ymm11,%ymm11 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti128 (%rcx),%ymm9 vbroadcasti128 (%r11),%ymm10 - +// Finish initializing LE_CTR by adding 1 to the second block. vpaddd .Lctr_pattern(%rip),%ymm11,%ymm11 - - +// If there are at least 128 bytes of data, then continue into the loop that +// processes 128 bytes of data at a time. Otherwise skip it. cmpq $127,%rdx jbe .Lcrypt_loop_4x_done__func1 vmovdqu 128(%r9),%ymm7 vmovdqu 128+32(%r9),%ymm8 - - - +// Encrypt the first 4 vectors of plaintext blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu .Linc_2blocks(%rip),%ymm2 vpshufb %ymm0,%ymm11,%ymm12 vpaddd %ymm2,%ymm11,%ymm11 @@ -423,7 +423,7 @@ vpshufb %ymm0,%ymm11,%ymm15 vpaddd %ymm2,%ymm11,%ymm11 - +// AES "round zero": XOR in the zero-th round key. vpxor %ymm9,%ymm12,%ymm12 vpxor %ymm9,%ymm13,%ymm13 vpxor %ymm9,%ymm14,%ymm14 @@ -453,16 +453,16 @@ vmovdqu %ymm14,64(%rsi) vmovdqu %ymm15,96(%rsi) - subq $-128,%rdi + subq $-128,%rdi // 128 is 4 bytes, -128 is 1 byte addq $-128,%rdx cmpq $127,%rdx jbe .Lghash_last_ciphertext_4x__func1 .align 16 .Lcrypt_loop_4x__func1: - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu .Linc_2blocks(%rip),%ymm2 vpshufb %ymm0,%ymm11,%ymm12 vpaddd %ymm2,%ymm11,%ymm11 @@ -473,7 +473,7 @@ vpshufb %ymm0,%ymm11,%ymm15 vpaddd %ymm2,%ymm11,%ymm11 - +// AES "round zero": XOR in the zero-th round key. vpxor %ymm9,%ymm12,%ymm12 vpxor %ymm9,%ymm13,%ymm13 vpxor %ymm9,%ymm14,%ymm14 @@ -482,7 +482,7 @@ cmpl $24,%r10d jl .Laes128__func1 je .Laes192__func1 - +// AES-256 vbroadcasti128 -208(%r11),%ymm2 vaesenc %ymm2,%ymm12,%ymm12 vaesenc %ymm2,%ymm13,%ymm13 @@ -511,7 +511,7 @@ .Laes128__func1: prefetcht0 512(%rdi) prefetcht0 512+64(%rdi) - +// First vector vmovdqu 0(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 0(%r9),%ymm4 @@ -535,7 +535,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Second vector vmovdqu 32(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 32(%r9),%ymm4 @@ -554,7 +554,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Third vector vmovdqu 64(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 64(%r9),%ymm4 @@ -581,7 +581,7 @@ vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Fourth vector vmovdqu 96(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 @@ -607,11 +607,11 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm1,%ymm6,%ymm6 - +// Fold lo into mi. vbroadcasti128 .Lgfpoly(%rip),%ymm4 vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 vpshufd $0x4e,%ymm5,%ymm5 @@ -624,7 +624,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 vpshufd $0x4e,%ymm6,%ymm6 vpxor %ymm6,%ymm1,%ymm1 @@ -640,7 +640,7 @@ vpxor %xmm2,%xmm1,%xmm1 - subq $-128,%rsi + subq $-128,%rsi // 128 is 4 bytes, -128 is 1 byte vpxor 0(%rdi),%ymm10,%ymm2 vpxor 32(%rdi),%ymm10,%ymm3 vpxor 64(%rdi),%ymm10,%ymm5 @@ -660,7 +660,7 @@ cmpq $127,%rdx ja .Lcrypt_loop_4x__func1 .Lghash_last_ciphertext_4x__func1: - +// First vector vmovdqu 0(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 0(%r9),%ymm4 @@ -670,7 +670,7 @@ vpunpckhqdq %ymm3,%ymm3,%ymm2 vpxor %ymm3,%ymm2,%ymm2 vpclmulqdq $0x00,%ymm7,%ymm2,%ymm6 - +// Second vector vmovdqu 32(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 32(%r9),%ymm4 @@ -682,7 +682,7 @@ vpxor %ymm3,%ymm2,%ymm2 vpclmulqdq $0x10,%ymm7,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Third vector vmovdqu 64(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 64(%r9),%ymm4 @@ -695,7 +695,7 @@ vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Fourth vector vmovdqu 96(%rsi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 96(%r9),%ymm4 @@ -707,17 +707,17 @@ vpxor %ymm3,%ymm2,%ymm2 vpclmulqdq $0x10,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm1,%ymm6,%ymm6 - +// Fold lo into mi. vbroadcasti128 .Lgfpoly(%rip),%ymm4 vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 vpshufd $0x4e,%ymm5,%ymm5 vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm2,%ymm6,%ymm6 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 vpshufd $0x4e,%ymm6,%ymm6 vpxor %ymm6,%ymm1,%ymm1 @@ -727,18 +727,18 @@ subq $-128,%rsi .Lcrypt_loop_4x_done__func1: - +// Check whether any data remains. testq %rdx,%rdx jz .Ldone__func1 +// DATALEN is in [16, 32, 48, 64, 80, 96, 112]. - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. leaq 128(%r9),%r8 subq %rdx,%r8 - +// Start collecting the unreduced GHASH intermediate value LO, MI, HI. vpxor %xmm5,%xmm5,%xmm5 vpxor %xmm6,%xmm6,%xmm6 vpxor %xmm7,%xmm7,%xmm7 @@ -746,7 +746,7 @@ cmpq $64,%rdx jb .Llessthan64bytes__func1 - +// DATALEN is in [64, 80, 96, 112]. Encrypt two vectors of counter blocks. vpshufb %ymm0,%ymm11,%ymm12 vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 vpshufb %ymm0,%ymm11,%ymm13 @@ -764,7 +764,7 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - +// XOR the data with the two vectors of keystream blocks. vmovdqu 0(%rdi),%ymm2 vmovdqu 32(%rdi),%ymm3 vpxor %ymm2,%ymm12,%ymm12 @@ -772,7 +772,7 @@ vmovdqu %ymm12,0(%rsi) vmovdqu %ymm13,32(%rsi) - +// Update GHASH with two vectors of ciphertext blocks, without reducing. vpshufb %ymm0,%ymm12,%ymm12 vpshufb %ymm0,%ymm13,%ymm13 vpxor %ymm1,%ymm12,%ymm12 @@ -800,7 +800,7 @@ vpxor %xmm1,%xmm1,%xmm1 - +// DATALEN is in [16, 32, 48]. Encrypt two last vectors of counter blocks. .Llessthan64bytes__func1: vpshufb %ymm0,%ymm11,%ymm12 vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 @@ -818,8 +818,8 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - - +// XOR the remaining data with the keystream blocks, and update GHASH with +// the remaining ciphertext blocks without reducing. cmpq $32,%rdx jb .Lxor_one_block__func1 @@ -876,7 +876,7 @@ vpxor %ymm4,%ymm7,%ymm7 .Lreduce__func1: - +// Finally, do the GHASH reduction. vbroadcasti128 .Lgfpoly(%rip),%ymm2 vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 vpshufd $0x4e,%ymm5,%ymm5 @@ -890,7 +890,7 @@ vpxor %xmm7,%xmm1,%xmm1 .Ldone__func1: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm0,%xmm1,%xmm1 vmovdqu %xmm1,(%r12) @@ -914,33 +914,33 @@ .cfi_adjust_cfa_offset 8 .cfi_offset %r12,-16 - movq 16(%rsp),%r12 + movq 16(%rsp),%r12 // arg7 vbroadcasti128 .Lbswap_mask(%rip),%ymm0 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm1 vpshufb %xmm0,%xmm1,%xmm1 vbroadcasti128 (%r8),%ymm11 vpshufb %ymm0,%ymm11,%ymm11 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti128 (%rcx),%ymm9 vbroadcasti128 (%r11),%ymm10 - +// Finish initializing LE_CTR by adding 1 to the second block. vpaddd .Lctr_pattern(%rip),%ymm11,%ymm11 - - +// If there are at least 128 bytes of data, then continue into the loop that +// processes 128 bytes of data at a time. Otherwise skip it. cmpq $127,%rdx jbe .Lcrypt_loop_4x_done__func2 @@ -949,9 +949,9 @@ .align 16 .Lcrypt_loop_4x__func2: - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu .Linc_2blocks(%rip),%ymm2 vpshufb %ymm0,%ymm11,%ymm12 vpaddd %ymm2,%ymm11,%ymm11 @@ -962,7 +962,7 @@ vpshufb %ymm0,%ymm11,%ymm15 vpaddd %ymm2,%ymm11,%ymm11 - +// AES "round zero": XOR in the zero-th round key. vpxor %ymm9,%ymm12,%ymm12 vpxor %ymm9,%ymm13,%ymm13 vpxor %ymm9,%ymm14,%ymm14 @@ -971,7 +971,7 @@ cmpl $24,%r10d jl .Laes128__func2 je .Laes192__func2 - +// AES-256 vbroadcasti128 -208(%r11),%ymm2 vaesenc %ymm2,%ymm12,%ymm12 vaesenc %ymm2,%ymm13,%ymm13 @@ -1000,7 +1000,7 @@ .Laes128__func2: prefetcht0 512(%rdi) prefetcht0 512+64(%rdi) - +// First vector vmovdqu 0(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 0(%r9),%ymm4 @@ -1024,7 +1024,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Second vector vmovdqu 32(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 32(%r9),%ymm4 @@ -1043,7 +1043,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Third vector vmovdqu 64(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 vmovdqu 64(%r9),%ymm4 @@ -1070,7 +1070,7 @@ vpclmulqdq $0x00,%ymm8,%ymm2,%ymm2 vpxor %ymm2,%ymm6,%ymm6 - +// Fourth vector vmovdqu 96(%rdi),%ymm3 vpshufb %ymm0,%ymm3,%ymm3 @@ -1096,11 +1096,11 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Finalize 'mi' following Karatsuba multiplication. vpxor %ymm5,%ymm6,%ymm6 vpxor %ymm1,%ymm6,%ymm6 - +// Fold lo into mi. vbroadcasti128 .Lgfpoly(%rip),%ymm4 vpclmulqdq $0x01,%ymm5,%ymm4,%ymm2 vpshufd $0x4e,%ymm5,%ymm5 @@ -1113,7 +1113,7 @@ vaesenc %ymm2,%ymm14,%ymm14 vaesenc %ymm2,%ymm15,%ymm15 - +// Fold mi into hi. vpclmulqdq $0x01,%ymm6,%ymm4,%ymm2 vpshufd $0x4e,%ymm6,%ymm6 vpxor %ymm6,%ymm1,%ymm1 @@ -1129,7 +1129,7 @@ vpxor %xmm2,%xmm1,%xmm1 - +// 128 is 4 bytes, -128 is 1 byte vpxor 0(%rdi),%ymm10,%ymm2 vpxor 32(%rdi),%ymm10,%ymm3 vpxor 64(%rdi),%ymm10,%ymm5 @@ -1149,18 +1149,18 @@ cmpq $127,%rdx ja .Lcrypt_loop_4x__func2 .Lcrypt_loop_4x_done__func2: - +// Check whether any data remains. testq %rdx,%rdx jz .Ldone__func2 +// DATALEN is in [16, 32, 48, 64, 80, 96, 112]. - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. leaq 128(%r9),%r8 subq %rdx,%r8 - +// Start collecting the unreduced GHASH intermediate value LO, MI, HI. vpxor %xmm5,%xmm5,%xmm5 vpxor %xmm6,%xmm6,%xmm6 vpxor %xmm7,%xmm7,%xmm7 @@ -1168,7 +1168,7 @@ cmpq $64,%rdx jb .Llessthan64bytes__func2 - +// DATALEN is in [64, 80, 96, 112]. Encrypt two vectors of counter blocks. vpshufb %ymm0,%ymm11,%ymm12 vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 vpshufb %ymm0,%ymm11,%ymm13 @@ -1186,7 +1186,7 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - +// XOR the data with the two vectors of keystream blocks. vmovdqu 0(%rdi),%ymm2 vmovdqu 32(%rdi),%ymm3 vpxor %ymm2,%ymm12,%ymm12 @@ -1194,7 +1194,7 @@ vmovdqu %ymm12,0(%rsi) vmovdqu %ymm13,32(%rsi) - +// Update GHASH with two vectors of ciphertext blocks, without reducing. vpshufb %ymm0,%ymm2,%ymm12 vpshufb %ymm0,%ymm3,%ymm13 vpxor %ymm1,%ymm12,%ymm12 @@ -1222,7 +1222,7 @@ vpxor %xmm1,%xmm1,%xmm1 - +// DATALEN is in [16, 32, 48]. Encrypt two last vectors of counter blocks. .Llessthan64bytes__func2: vpshufb %ymm0,%ymm11,%ymm12 vpaddd .Linc_2blocks(%rip),%ymm11,%ymm11 @@ -1240,8 +1240,8 @@ vaesenclast %ymm10,%ymm12,%ymm12 vaesenclast %ymm10,%ymm13,%ymm13 - - +// XOR the remaining data with the keystream blocks, and update GHASH with +// the remaining ciphertext blocks without reducing. cmpq $32,%rdx jb .Lxor_one_block__func2 @@ -1298,7 +1298,7 @@ vpxor %ymm4,%ymm7,%ymm7 .Lreduce__func2: - +// Finally, do the GHASH reduction. vbroadcasti128 .Lgfpoly(%rip),%ymm2 vpclmulqdq $0x01,%ymm5,%ymm2,%ymm3 vpshufd $0x4e,%ymm5,%ymm5 @@ -1312,7 +1312,7 @@ vpxor %xmm7,%xmm1,%xmm1 .Ldone__func2: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm0,%xmm1,%xmm1 vmovdqu %xmm1,(%r12)
diff --git a/gen/bcm/aes-gcm-avx2-x86_64-win.asm b/gen/bcm/aes-gcm-avx2-x86_64-win.asm index b50aba7..1bd096e 100644 --- a/gen/bcm/aes-gcm-avx2-x86_64-win.asm +++ b/gen/bcm/aes-gcm-avx2-x86_64-win.asm
@@ -16,26 +16,26 @@ ALIGN 16 aes_gcm_avx2_constants: - +; A shuffle mask that reflects the bytes of 16-byte blocks $L$bswap_mask: DQ 0x08090a0b0c0d0e0f,0x0001020304050607 - - - - - - - +; This is the GHASH reducing polynomial without its constant term, i.e. +; x^128 + x^7 + x^2 + x, represented using the backwards mapping +; between bits and polynomial coefficients. +; +; Alternatively, it can be interpreted as the naturally-ordered +; representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the +; "reversed" GHASH reducing polynomial without its x^128 term. $L$gfpoly: DQ 1,0xc200000000000000 - +; Same as above, but with the (1 << 64) bit set. $L$gfpoly_and_internal_carrybit: DQ 1,0xc200000000000001 ALIGN 32 - +; The below constants are used for incrementing the counter blocks. $L$ctr_pattern: DQ 0,0 DQ 1,0 @@ -59,14 +59,14 @@ $L$SEH_endprologue_gcm_init_vpclmulqdq_avx2_4: - - +; Load the byte-reflected hash subkey. BoringSSL provides it in +; byte-reflected form except the two halves are in the wrong order. vpshufd xmm3,XMMWORD[rdx],0x4e - - - - +; Finish preprocessing the byte-reflected hash subkey by multiplying it by +; x^-1 ("standard" interpretation of polynomial coefficients) or +; equivalently x^1 (natural interpretation). This gets the key into a +; format that avoids having to bit-reflect the data blocks later. vpshufd xmm0,xmm3,0xd3 vpsrad xmm0,xmm0,31 vpaddq xmm3,xmm3,xmm3 @@ -75,83 +75,83 @@ vbroadcasti128 ymm6,XMMWORD[$L$gfpoly] - - vpclmulqdq xmm0,xmm3,xmm3,0x00 - vpclmulqdq xmm5,xmm3,xmm3,0x11 - vpclmulqdq xmm1,xmm6,xmm0,0x01 - vpshufd xmm0,xmm0,0x4e - vpxor xmm1,xmm1,xmm0 - vpclmulqdq xmm0,xmm6,xmm1,0x01 - vpshufd xmm1,xmm1,0x4e - vpxor xmm5,xmm5,xmm1 - vpxor xmm5,xmm5,xmm0 +; Square H^1 to get H^2. + vpclmulqdq xmm0,xmm3,xmm3,0x00 ; LO = a_L * a_L + vpclmulqdq xmm5,xmm3,xmm3,0x11 ; HI = a_H * a_H + vpclmulqdq xmm1,xmm6,xmm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd xmm0,xmm0,0x4e ; Swap halves of LO + vpxor xmm1,xmm1,xmm0 ; Fold LO into MI + vpclmulqdq xmm0,xmm6,xmm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd xmm1,xmm1,0x4e ; Swap halves of MI + vpxor xmm5,xmm5,xmm1 ; Fold MI into HI (part 1) + vpxor xmm5,xmm5,xmm0 ; Fold MI into HI (part 2) - +; Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2]. vinserti128 ymm3,ymm5,xmm3,1 vinserti128 ymm5,ymm5,xmm5,1 - - vpclmulqdq ymm0,ymm3,ymm5,0x00 - vpclmulqdq ymm1,ymm3,ymm5,0x01 - vpclmulqdq ymm2,ymm3,ymm5,0x10 - vpxor ymm1,ymm1,ymm2 - vpclmulqdq ymm2,ymm6,ymm0,0x01 - vpshufd ymm0,ymm0,0x4e - vpxor ymm1,ymm1,ymm0 - vpxor ymm1,ymm1,ymm2 - vpclmulqdq ymm4,ymm3,ymm5,0x11 - vpclmulqdq ymm0,ymm6,ymm1,0x01 - vpshufd ymm1,ymm1,0x4e - vpxor ymm4,ymm4,ymm1 - vpxor ymm4,ymm4,ymm0 +; Compute H_CUR2 = [H^4, H^3]. + vpclmulqdq ymm0,ymm3,ymm5,0x00 ; LO = a_L * b_L + vpclmulqdq ymm1,ymm3,ymm5,0x01 ; MI_0 = a_L * b_H + vpclmulqdq ymm2,ymm3,ymm5,0x10 ; MI_1 = a_H * b_L + vpxor ymm1,ymm1,ymm2 ; MI = MI_0 + MI_1 + vpclmulqdq ymm2,ymm6,ymm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd ymm0,ymm0,0x4e ; Swap halves of LO + vpxor ymm1,ymm1,ymm0 ; Fold LO into MI (part 1) + vpxor ymm1,ymm1,ymm2 ; Fold LO into MI (part 2) + vpclmulqdq ymm4,ymm3,ymm5,0x11 ; HI = a_H * b_H + vpclmulqdq ymm0,ymm6,ymm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd ymm1,ymm1,0x4e ; Swap halves of MI + vpxor ymm4,ymm4,ymm1 ; Fold MI into HI (part 1) + vpxor ymm4,ymm4,ymm0 ; Fold MI into HI (part 2) - +; Store [H^2, H^1] and [H^4, H^3]. vmovdqu YMMWORD[96+rcx],ymm3 vmovdqu YMMWORD[64+rcx],ymm4 - - +; For Karatsuba multiplication: compute and store the two 64-bit halves of +; each key power XOR'd together. Order is 4,2,3,1. vpunpcklqdq ymm0,ymm4,ymm3 vpunpckhqdq ymm1,ymm4,ymm3 vpxor ymm0,ymm0,ymm1 vmovdqu YMMWORD[(128+32)+rcx],ymm0 +; Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7]. + vpclmulqdq ymm0,ymm4,ymm5,0x00 ; LO = a_L * b_L + vpclmulqdq ymm1,ymm4,ymm5,0x01 ; MI_0 = a_L * b_H + vpclmulqdq ymm2,ymm4,ymm5,0x10 ; MI_1 = a_H * b_L + vpxor ymm1,ymm1,ymm2 ; MI = MI_0 + MI_1 + vpclmulqdq ymm2,ymm6,ymm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd ymm0,ymm0,0x4e ; Swap halves of LO + vpxor ymm1,ymm1,ymm0 ; Fold LO into MI (part 1) + vpxor ymm1,ymm1,ymm2 ; Fold LO into MI (part 2) + vpclmulqdq ymm3,ymm4,ymm5,0x11 ; HI = a_H * b_H + vpclmulqdq ymm0,ymm6,ymm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd ymm1,ymm1,0x4e ; Swap halves of MI + vpxor ymm3,ymm3,ymm1 ; Fold MI into HI (part 1) + vpxor ymm3,ymm3,ymm0 ; Fold MI into HI (part 2) - vpclmulqdq ymm0,ymm4,ymm5,0x00 - vpclmulqdq ymm1,ymm4,ymm5,0x01 - vpclmulqdq ymm2,ymm4,ymm5,0x10 - vpxor ymm1,ymm1,ymm2 - vpclmulqdq ymm2,ymm6,ymm0,0x01 - vpshufd ymm0,ymm0,0x4e - vpxor ymm1,ymm1,ymm0 - vpxor ymm1,ymm1,ymm2 - vpclmulqdq ymm3,ymm4,ymm5,0x11 - vpclmulqdq ymm0,ymm6,ymm1,0x01 - vpshufd ymm1,ymm1,0x4e - vpxor ymm3,ymm3,ymm1 - vpxor ymm3,ymm3,ymm0 - - vpclmulqdq ymm0,ymm3,ymm5,0x00 - vpclmulqdq ymm1,ymm3,ymm5,0x01 - vpclmulqdq ymm2,ymm3,ymm5,0x10 - vpxor ymm1,ymm1,ymm2 - vpclmulqdq ymm2,ymm6,ymm0,0x01 - vpshufd ymm0,ymm0,0x4e - vpxor ymm1,ymm1,ymm0 - vpxor ymm1,ymm1,ymm2 - vpclmulqdq ymm4,ymm3,ymm5,0x11 - vpclmulqdq ymm0,ymm6,ymm1,0x01 - vpshufd ymm1,ymm1,0x4e - vpxor ymm4,ymm4,ymm1 - vpxor ymm4,ymm4,ymm0 + vpclmulqdq ymm0,ymm3,ymm5,0x00 ; LO = a_L * b_L + vpclmulqdq ymm1,ymm3,ymm5,0x01 ; MI_0 = a_L * b_H + vpclmulqdq ymm2,ymm3,ymm5,0x10 ; MI_1 = a_H * b_L + vpxor ymm1,ymm1,ymm2 ; MI = MI_0 + MI_1 + vpclmulqdq ymm2,ymm6,ymm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd ymm0,ymm0,0x4e ; Swap halves of LO + vpxor ymm1,ymm1,ymm0 ; Fold LO into MI (part 1) + vpxor ymm1,ymm1,ymm2 ; Fold LO into MI (part 2) + vpclmulqdq ymm4,ymm3,ymm5,0x11 ; HI = a_H * b_H + vpclmulqdq ymm0,ymm6,ymm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd ymm1,ymm1,0x4e ; Swap halves of MI + vpxor ymm4,ymm4,ymm1 ; Fold MI into HI (part 1) + vpxor ymm4,ymm4,ymm0 ; Fold MI into HI (part 2) vmovdqu YMMWORD[32+rcx],ymm3 vmovdqu YMMWORD[rcx],ymm4 - - +; Again, compute and store the two 64-bit halves of each key power XOR'd +; together. Order is 8,6,7,5. vpunpcklqdq ymm0,ymm4,ymm3 vpunpckhqdq ymm1,ymm4,ymm3 vpxor ymm0,ymm0,ymm1 @@ -184,25 +184,25 @@ vmovdqu xmm3,XMMWORD[$L$gfpoly] vpshufb xmm0,xmm0,xmm1 - vpclmulqdq xmm4,xmm0,xmm2,0x00 - vpclmulqdq xmm5,xmm0,xmm2,0x01 - vpclmulqdq xmm6,xmm0,xmm2,0x10 - vpxor xmm5,xmm5,xmm6 - vpclmulqdq xmm6,xmm3,xmm4,0x01 - vpshufd xmm4,xmm4,0x4e - vpxor xmm5,xmm5,xmm4 - vpxor xmm5,xmm5,xmm6 - vpclmulqdq xmm0,xmm0,xmm2,0x11 - vpclmulqdq xmm4,xmm3,xmm5,0x01 - vpshufd xmm5,xmm5,0x4e - vpxor xmm0,xmm0,xmm5 - vpxor xmm0,xmm0,xmm4 + vpclmulqdq xmm4,xmm0,xmm2,0x00 ; LO = a_L * b_L + vpclmulqdq xmm5,xmm0,xmm2,0x01 ; MI_0 = a_L * b_H + vpclmulqdq xmm6,xmm0,xmm2,0x10 ; MI_1 = a_H * b_L + vpxor xmm5,xmm5,xmm6 ; MI = MI_0 + MI_1 + vpclmulqdq xmm6,xmm3,xmm4,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd xmm4,xmm4,0x4e ; Swap halves of LO + vpxor xmm5,xmm5,xmm4 ; Fold LO into MI (part 1) + vpxor xmm5,xmm5,xmm6 ; Fold LO into MI (part 2) + vpclmulqdq xmm0,xmm0,xmm2,0x11 ; HI = a_H * b_H + vpclmulqdq xmm4,xmm3,xmm5,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd xmm5,xmm5,0x4e ; Swap halves of MI + vpxor xmm0,xmm0,xmm5 ; Fold MI into HI (part 1) + vpxor xmm0,xmm0,xmm4 ; Fold MI into HI (part 2) vpshufb xmm0,xmm0,xmm1 vmovdqu XMMWORD[rcx],xmm0 - +; No need for vzeroupper, since only xmm registers were used. vmovdqa xmm6,XMMWORD[rsp] add rsp,24 ret @@ -229,33 +229,33 @@ $L$SEH_endprologue_gcm_ghash_vpclmulqdq_avx2_7: - - - +; Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, +; usually only 128-bit vectors will be used. So as an optimization, don't +; broadcast these constants to both 128-bit lanes quite yet. vmovdqu xmm6,XMMWORD[$L$bswap_mask] vmovdqu xmm7,XMMWORD[$L$gfpoly] - +; Load the GHASH accumulator. vmovdqu xmm5,XMMWORD[rcx] vpshufb xmm5,xmm5,xmm6 - +; Optimize for AADLEN < 32 by checking for AADLEN < 32 before AADLEN < 128. cmp r9,32 jb NEAR $L$ghash_lastblock - - +; AADLEN >= 32, so we'll operate on full vectors. Broadcast bswap_mask and +; gfpoly to both 128-bit lanes. vinserti128 ymm6,ymm6,xmm6,1 vinserti128 ymm7,ymm7,xmm7,1 cmp r9,127 jbe NEAR $L$ghash_loop_1x - +; Update GHASH with 128 bytes of AAD at a time. vmovdqu ymm8,YMMWORD[128+rdx] vmovdqu ymm9,YMMWORD[((128+32))+rdx] $L$ghash_loop_4x: - +; First vector vmovdqu ymm1,YMMWORD[r8] vpshufb ymm1,ymm1,ymm6 vmovdqu ymm2,YMMWORD[rdx] @@ -265,7 +265,7 @@ vpunpckhqdq ymm0,ymm1,ymm1 vpxor ymm0,ymm0,ymm1 vpclmulqdq ymm4,ymm0,ymm8,0x00 - +; Second vector vmovdqu ymm1,YMMWORD[32+r8] vpshufb ymm1,ymm1,ymm6 vmovdqu ymm2,YMMWORD[32+rdx] @@ -277,7 +277,7 @@ vpxor ymm0,ymm0,ymm1 vpclmulqdq ymm0,ymm0,ymm8,0x10 vpxor ymm4,ymm4,ymm0 - +; Third vector vmovdqu ymm1,YMMWORD[64+r8] vpshufb ymm1,ymm1,ymm6 vmovdqu ymm2,YMMWORD[64+rdx] @@ -290,7 +290,7 @@ vpclmulqdq ymm0,ymm0,ymm9,0x00 vpxor ymm4,ymm4,ymm0 - +; Fourth vector vmovdqu ymm1,YMMWORD[96+r8] vpshufb ymm1,ymm1,ymm6 vmovdqu ymm2,YMMWORD[96+rdx] @@ -302,17 +302,17 @@ vpxor ymm0,ymm0,ymm1 vpclmulqdq ymm0,ymm0,ymm9,0x10 vpxor ymm4,ymm4,ymm0 - +; Finalize 'mi' following Karatsuba multiplication. vpxor ymm4,ymm4,ymm3 vpxor ymm4,ymm4,ymm5 - +; Fold lo into mi. vbroadcasti128 ymm2,XMMWORD[$L$gfpoly] vpclmulqdq ymm0,ymm2,ymm3,0x01 vpshufd ymm3,ymm3,0x4e vpxor ymm4,ymm4,ymm3 vpxor ymm4,ymm4,ymm0 - +; Fold mi into hi. vpclmulqdq ymm0,ymm2,ymm4,0x01 vpshufd ymm4,ymm4,0x4e vpxor ymm5,ymm5,ymm4 @@ -320,12 +320,12 @@ vextracti128 xmm0,ymm5,1 vpxor xmm5,xmm5,xmm0 - sub r8,-128 + sub r8,-128 ; 128 is 4 bytes, -128 is 1 byte add r9,-128 cmp r9,127 ja NEAR $L$ghash_loop_4x - +; Update GHASH with 32 bytes of AAD at a time. cmp r9,32 jb NEAR $L$ghash_loop_1x_done $L$ghash_loop_1x: @@ -333,19 +333,19 @@ vpshufb ymm0,ymm0,ymm6 vpxor ymm5,ymm5,ymm0 vmovdqu ymm0,YMMWORD[((128-32))+rdx] - vpclmulqdq ymm1,ymm5,ymm0,0x00 - vpclmulqdq ymm2,ymm5,ymm0,0x01 - vpclmulqdq ymm3,ymm5,ymm0,0x10 - vpxor ymm2,ymm2,ymm3 - vpclmulqdq ymm3,ymm7,ymm1,0x01 - vpshufd ymm1,ymm1,0x4e - vpxor ymm2,ymm2,ymm1 - vpxor ymm2,ymm2,ymm3 - vpclmulqdq ymm5,ymm5,ymm0,0x11 - vpclmulqdq ymm1,ymm7,ymm2,0x01 - vpshufd ymm2,ymm2,0x4e - vpxor ymm5,ymm5,ymm2 - vpxor ymm5,ymm5,ymm1 + vpclmulqdq ymm1,ymm5,ymm0,0x00 ; LO = a_L * b_L + vpclmulqdq ymm2,ymm5,ymm0,0x01 ; MI_0 = a_L * b_H + vpclmulqdq ymm3,ymm5,ymm0,0x10 ; MI_1 = a_H * b_L + vpxor ymm2,ymm2,ymm3 ; MI = MI_0 + MI_1 + vpclmulqdq ymm3,ymm7,ymm1,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd ymm1,ymm1,0x4e ; Swap halves of LO + vpxor ymm2,ymm2,ymm1 ; Fold LO into MI (part 1) + vpxor ymm2,ymm2,ymm3 ; Fold LO into MI (part 2) + vpclmulqdq ymm5,ymm5,ymm0,0x11 ; HI = a_H * b_H + vpclmulqdq ymm1,ymm7,ymm2,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd ymm2,ymm2,0x4e ; Swap halves of MI + vpxor ymm5,ymm5,ymm2 ; Fold MI into HI (part 1) + vpxor ymm5,ymm5,ymm1 ; Fold MI into HI (part 2) vextracti128 xmm0,ymm5,1 vpxor xmm5,xmm5,xmm0 @@ -355,7 +355,7 @@ jae NEAR $L$ghash_loop_1x $L$ghash_loop_1x_done: - +; Update GHASH with the remaining 16-byte block if any. $L$ghash_lastblock: test r9,r9 jz NEAR $L$ghash_done @@ -363,23 +363,23 @@ vpshufb xmm0,xmm0,xmm6 vpxor xmm5,xmm5,xmm0 vmovdqu xmm0,XMMWORD[((128-16))+rdx] - vpclmulqdq xmm1,xmm5,xmm0,0x00 - vpclmulqdq xmm2,xmm5,xmm0,0x01 - vpclmulqdq xmm3,xmm5,xmm0,0x10 - vpxor xmm2,xmm2,xmm3 - vpclmulqdq xmm3,xmm7,xmm1,0x01 - vpshufd xmm1,xmm1,0x4e - vpxor xmm2,xmm2,xmm1 - vpxor xmm2,xmm2,xmm3 - vpclmulqdq xmm5,xmm5,xmm0,0x11 - vpclmulqdq xmm1,xmm7,xmm2,0x01 - vpshufd xmm2,xmm2,0x4e - vpxor xmm5,xmm5,xmm2 - vpxor xmm5,xmm5,xmm1 + vpclmulqdq xmm1,xmm5,xmm0,0x00 ; LO = a_L * b_L + vpclmulqdq xmm2,xmm5,xmm0,0x01 ; MI_0 = a_L * b_H + vpclmulqdq xmm3,xmm5,xmm0,0x10 ; MI_1 = a_H * b_L + vpxor xmm2,xmm2,xmm3 ; MI = MI_0 + MI_1 + vpclmulqdq xmm3,xmm7,xmm1,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd xmm1,xmm1,0x4e ; Swap halves of LO + vpxor xmm2,xmm2,xmm1 ; Fold LO into MI (part 1) + vpxor xmm2,xmm2,xmm3 ; Fold LO into MI (part 2) + vpclmulqdq xmm5,xmm5,xmm0,0x11 ; HI = a_H * b_H + vpclmulqdq xmm1,xmm7,xmm2,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd xmm2,xmm2,0x4e ; Swap halves of MI + vpxor xmm5,xmm5,xmm2 ; Fold MI into HI (part 1) + vpxor xmm5,xmm5,xmm1 ; Fold MI into HI (part 2) $L$ghash_done: - +; Store the updated GHASH accumulator back to memory. vpshufb xmm5,xmm5,xmm6 vmovdqu XMMWORD[rcx],xmm5 @@ -407,9 +407,9 @@ push r12 $L$SEH_prologue_aes_gcm_enc_update_vaes_avx2_4: - mov rsi,QWORD[64+rsp] - mov rdi,QWORD[72+rsp] - mov r12,QWORD[80+rsp] + mov rsi,QWORD[64+rsp] ; arg5 + mov rdi,QWORD[72+rsp] ; arg6 + mov r12,QWORD[80+rsp] ; arg7 sub rsp,160 $L$SEH_prologue_aes_gcm_enc_update_vaes_avx2_5: vmovdqa XMMWORD[rsp],xmm6 @@ -440,38 +440,38 @@ %endif vbroadcasti128 ymm0,XMMWORD[$L$bswap_mask] - - +; Load the GHASH accumulator and the starting counter. +; BoringSSL passes these values in big endian format. vmovdqu xmm1,XMMWORD[r12] vpshufb xmm1,xmm1,xmm0 vbroadcasti128 ymm11,XMMWORD[rsi] vpshufb ymm11,ymm11,ymm0 - - +; Load the AES key length in bytes. BoringSSL stores number of rounds +; minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. mov r10d,DWORD[240+r9] lea r10d,[((-20))+r10*4] - - - +; Make RNDKEYLAST_PTR point to the last AES round key. This is the +; round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +; respectively. Then load the zero-th and last round keys. lea r11,[96+r10*4+r9] vbroadcasti128 ymm9,XMMWORD[r9] vbroadcasti128 ymm10,XMMWORD[r11] - +; Finish initializing LE_CTR by adding 1 to the second block. vpaddd ymm11,ymm11,YMMWORD[$L$ctr_pattern] - - +; If there are at least 128 bytes of data, then continue into the loop that +; processes 128 bytes of data at a time. Otherwise skip it. cmp r8,127 jbe NEAR $L$crypt_loop_4x_done__func1 vmovdqu ymm7,YMMWORD[128+rdi] vmovdqu ymm8,YMMWORD[((128+32))+rdi] - - - +; Encrypt the first 4 vectors of plaintext blocks. +; Increment le_ctr four times to generate four vectors of little-endian +; counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu ymm2,YMMWORD[$L$inc_2blocks] vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,ymm2 @@ -482,7 +482,7 @@ vpshufb ymm15,ymm11,ymm0 vpaddd ymm11,ymm11,ymm2 - +; AES "round zero": XOR in the zero-th round key. vpxor ymm12,ymm12,ymm9 vpxor ymm13,ymm13,ymm9 vpxor ymm14,ymm14,ymm9 @@ -512,16 +512,16 @@ vmovdqu YMMWORD[64+rdx],ymm14 vmovdqu YMMWORD[96+rdx],ymm15 - sub rcx,-128 + sub rcx,-128 ; 128 is 4 bytes, -128 is 1 byte add r8,-128 cmp r8,127 jbe NEAR $L$ghash_last_ciphertext_4x__func1 ALIGN 16 $L$crypt_loop_4x__func1: - - - +; Start the AES encryption of the counter blocks. +; Increment le_ctr four times to generate four vectors of little-endian +; counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu ymm2,YMMWORD[$L$inc_2blocks] vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,ymm2 @@ -532,7 +532,7 @@ vpshufb ymm15,ymm11,ymm0 vpaddd ymm11,ymm11,ymm2 - +; AES "round zero": XOR in the zero-th round key. vpxor ymm12,ymm12,ymm9 vpxor ymm13,ymm13,ymm9 vpxor ymm14,ymm14,ymm9 @@ -541,7 +541,7 @@ cmp r10d,24 jl NEAR $L$aes128__func1 je NEAR $L$aes192__func1 - +; AES-256 vbroadcasti128 ymm2,XMMWORD[((-208))+r11] vaesenc ymm12,ymm12,ymm2 vaesenc ymm13,ymm13,ymm2 @@ -570,7 +570,7 @@ $L$aes128__func1: prefetcht0 [512+rcx] prefetcht0 [((512+64))+rcx] - +; First vector vmovdqu ymm3,YMMWORD[rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[rdi] @@ -594,7 +594,7 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Second vector vmovdqu ymm3,YMMWORD[32+rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[32+rdi] @@ -613,7 +613,7 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Third vector vmovdqu ymm3,YMMWORD[64+rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[64+rdi] @@ -640,7 +640,7 @@ vpclmulqdq ymm2,ymm2,ymm8,0x00 vpxor ymm6,ymm6,ymm2 - +; Fourth vector vmovdqu ymm3,YMMWORD[96+rdx] vpshufb ymm3,ymm3,ymm0 @@ -666,11 +666,11 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Finalize 'mi' following Karatsuba multiplication. vpxor ymm6,ymm6,ymm5 vpxor ymm6,ymm6,ymm1 - +; Fold lo into mi. vbroadcasti128 ymm4,XMMWORD[$L$gfpoly] vpclmulqdq ymm2,ymm4,ymm5,0x01 vpshufd ymm5,ymm5,0x4e @@ -683,7 +683,7 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Fold mi into hi. vpclmulqdq ymm2,ymm4,ymm6,0x01 vpshufd ymm6,ymm6,0x4e vpxor ymm1,ymm1,ymm6 @@ -699,7 +699,7 @@ vpxor xmm1,xmm1,xmm2 - sub rdx,-128 + sub rdx,-128 ; 128 is 4 bytes, -128 is 1 byte vpxor ymm2,ymm10,YMMWORD[rcx] vpxor ymm3,ymm10,YMMWORD[32+rcx] vpxor ymm5,ymm10,YMMWORD[64+rcx] @@ -719,7 +719,7 @@ cmp r8,127 ja NEAR $L$crypt_loop_4x__func1 $L$ghash_last_ciphertext_4x__func1: - +; First vector vmovdqu ymm3,YMMWORD[rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[rdi] @@ -729,7 +729,7 @@ vpunpckhqdq ymm2,ymm3,ymm3 vpxor ymm2,ymm2,ymm3 vpclmulqdq ymm6,ymm2,ymm7,0x00 - +; Second vector vmovdqu ymm3,YMMWORD[32+rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[32+rdi] @@ -741,7 +741,7 @@ vpxor ymm2,ymm2,ymm3 vpclmulqdq ymm2,ymm2,ymm7,0x10 vpxor ymm6,ymm6,ymm2 - +; Third vector vmovdqu ymm3,YMMWORD[64+rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[64+rdi] @@ -754,7 +754,7 @@ vpclmulqdq ymm2,ymm2,ymm8,0x00 vpxor ymm6,ymm6,ymm2 - +; Fourth vector vmovdqu ymm3,YMMWORD[96+rdx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[96+rdi] @@ -766,17 +766,17 @@ vpxor ymm2,ymm2,ymm3 vpclmulqdq ymm2,ymm2,ymm8,0x10 vpxor ymm6,ymm6,ymm2 - +; Finalize 'mi' following Karatsuba multiplication. vpxor ymm6,ymm6,ymm5 vpxor ymm6,ymm6,ymm1 - +; Fold lo into mi. vbroadcasti128 ymm4,XMMWORD[$L$gfpoly] vpclmulqdq ymm2,ymm4,ymm5,0x01 vpshufd ymm5,ymm5,0x4e vpxor ymm6,ymm6,ymm5 vpxor ymm6,ymm6,ymm2 - +; Fold mi into hi. vpclmulqdq ymm2,ymm4,ymm6,0x01 vpshufd ymm6,ymm6,0x4e vpxor ymm1,ymm1,ymm6 @@ -786,18 +786,18 @@ sub rdx,-128 $L$crypt_loop_4x_done__func1: - +; Check whether any data remains. test r8,r8 jz NEAR $L$done__func1 +; DATALEN is in [16, 32, 48, 64, 80, 96, 112]. - - - +; Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +; is the number of blocks that remain. lea rsi,[128+rdi] sub rsi,r8 - +; Start collecting the unreduced GHASH intermediate value LO, MI, HI. vpxor xmm5,xmm5,xmm5 vpxor xmm6,xmm6,xmm6 vpxor xmm7,xmm7,xmm7 @@ -805,7 +805,7 @@ cmp r8,64 jb NEAR $L$lessthan64bytes__func1 - +; DATALEN is in [64, 80, 96, 112]. Encrypt two vectors of counter blocks. vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,YMMWORD[$L$inc_2blocks] vpshufb ymm13,ymm11,ymm0 @@ -823,7 +823,7 @@ vaesenclast ymm12,ymm12,ymm10 vaesenclast ymm13,ymm13,ymm10 - +; XOR the data with the two vectors of keystream blocks. vmovdqu ymm2,YMMWORD[rcx] vmovdqu ymm3,YMMWORD[32+rcx] vpxor ymm12,ymm12,ymm2 @@ -831,7 +831,7 @@ vmovdqu YMMWORD[rdx],ymm12 vmovdqu YMMWORD[32+rdx],ymm13 - +; Update GHASH with two vectors of ciphertext blocks, without reducing. vpshufb ymm12,ymm12,ymm0 vpshufb ymm13,ymm13,ymm0 vpxor ymm12,ymm12,ymm1 @@ -859,7 +859,7 @@ vpxor xmm1,xmm1,xmm1 - +; DATALEN is in [16, 32, 48]. Encrypt two last vectors of counter blocks. $L$lessthan64bytes__func1: vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,YMMWORD[$L$inc_2blocks] @@ -877,8 +877,8 @@ vaesenclast ymm12,ymm12,ymm10 vaesenclast ymm13,ymm13,ymm10 - - +; XOR the remaining data with the keystream blocks, and update GHASH with +; the remaining ciphertext blocks without reducing. cmp r8,32 jb NEAR $L$xor_one_block__func1 @@ -935,7 +935,7 @@ vpxor ymm7,ymm7,ymm4 $L$reduce__func1: - +; Finally, do the GHASH reduction. vbroadcasti128 ymm2,XMMWORD[$L$gfpoly] vpclmulqdq ymm3,ymm2,ymm5,0x01 vpshufd ymm5,ymm5,0x4e @@ -949,7 +949,7 @@ vpxor xmm1,xmm1,xmm7 $L$done__func1: - +; Store the updated GHASH accumulator back to memory. vpshufb xmm1,xmm1,xmm0 vmovdqu XMMWORD[r12],xmm1 @@ -986,9 +986,9 @@ push r12 $L$SEH_prologue_aes_gcm_dec_update_vaes_avx2_4: - mov rsi,QWORD[64+rsp] - mov rdi,QWORD[72+rsp] - mov r12,QWORD[80+rsp] + mov rsi,QWORD[64+rsp] ; arg5 + mov rdi,QWORD[72+rsp] ; arg6 + mov r12,QWORD[80+rsp] ; arg7 sub rsp,160 $L$SEH_prologue_aes_gcm_dec_update_vaes_avx2_5: vmovdqa XMMWORD[rsp],xmm6 @@ -1015,30 +1015,30 @@ $L$SEH_endprologue_aes_gcm_dec_update_vaes_avx2_16: vbroadcasti128 ymm0,XMMWORD[$L$bswap_mask] - - +; Load the GHASH accumulator and the starting counter. +; BoringSSL passes these values in big endian format. vmovdqu xmm1,XMMWORD[r12] vpshufb xmm1,xmm1,xmm0 vbroadcasti128 ymm11,XMMWORD[rsi] vpshufb ymm11,ymm11,ymm0 - - +; Load the AES key length in bytes. BoringSSL stores number of rounds +; minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. mov r10d,DWORD[240+r9] lea r10d,[((-20))+r10*4] - - - +; Make RNDKEYLAST_PTR point to the last AES round key. This is the +; round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +; respectively. Then load the zero-th and last round keys. lea r11,[96+r10*4+r9] vbroadcasti128 ymm9,XMMWORD[r9] vbroadcasti128 ymm10,XMMWORD[r11] - +; Finish initializing LE_CTR by adding 1 to the second block. vpaddd ymm11,ymm11,YMMWORD[$L$ctr_pattern] - - +; If there are at least 128 bytes of data, then continue into the loop that +; processes 128 bytes of data at a time. Otherwise skip it. cmp r8,127 jbe NEAR $L$crypt_loop_4x_done__func2 @@ -1047,9 +1047,9 @@ ALIGN 16 $L$crypt_loop_4x__func2: - - - +; Start the AES encryption of the counter blocks. +; Increment le_ctr four times to generate four vectors of little-endian +; counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vmovdqu ymm2,YMMWORD[$L$inc_2blocks] vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,ymm2 @@ -1060,7 +1060,7 @@ vpshufb ymm15,ymm11,ymm0 vpaddd ymm11,ymm11,ymm2 - +; AES "round zero": XOR in the zero-th round key. vpxor ymm12,ymm12,ymm9 vpxor ymm13,ymm13,ymm9 vpxor ymm14,ymm14,ymm9 @@ -1069,7 +1069,7 @@ cmp r10d,24 jl NEAR $L$aes128__func2 je NEAR $L$aes192__func2 - +; AES-256 vbroadcasti128 ymm2,XMMWORD[((-208))+r11] vaesenc ymm12,ymm12,ymm2 vaesenc ymm13,ymm13,ymm2 @@ -1098,7 +1098,7 @@ $L$aes128__func2: prefetcht0 [512+rcx] prefetcht0 [((512+64))+rcx] - +; First vector vmovdqu ymm3,YMMWORD[rcx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[rdi] @@ -1122,7 +1122,7 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Second vector vmovdqu ymm3,YMMWORD[32+rcx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[32+rdi] @@ -1141,7 +1141,7 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Third vector vmovdqu ymm3,YMMWORD[64+rcx] vpshufb ymm3,ymm3,ymm0 vmovdqu ymm4,YMMWORD[64+rdi] @@ -1168,7 +1168,7 @@ vpclmulqdq ymm2,ymm2,ymm8,0x00 vpxor ymm6,ymm6,ymm2 - +; Fourth vector vmovdqu ymm3,YMMWORD[96+rcx] vpshufb ymm3,ymm3,ymm0 @@ -1194,11 +1194,11 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Finalize 'mi' following Karatsuba multiplication. vpxor ymm6,ymm6,ymm5 vpxor ymm6,ymm6,ymm1 - +; Fold lo into mi. vbroadcasti128 ymm4,XMMWORD[$L$gfpoly] vpclmulqdq ymm2,ymm4,ymm5,0x01 vpshufd ymm5,ymm5,0x4e @@ -1211,7 +1211,7 @@ vaesenc ymm14,ymm14,ymm2 vaesenc ymm15,ymm15,ymm2 - +; Fold mi into hi. vpclmulqdq ymm2,ymm4,ymm6,0x01 vpshufd ymm6,ymm6,0x4e vpxor ymm1,ymm1,ymm6 @@ -1227,7 +1227,7 @@ vpxor xmm1,xmm1,xmm2 - +; 128 is 4 bytes, -128 is 1 byte vpxor ymm2,ymm10,YMMWORD[rcx] vpxor ymm3,ymm10,YMMWORD[32+rcx] vpxor ymm5,ymm10,YMMWORD[64+rcx] @@ -1247,18 +1247,18 @@ cmp r8,127 ja NEAR $L$crypt_loop_4x__func2 $L$crypt_loop_4x_done__func2: - +; Check whether any data remains. test r8,r8 jz NEAR $L$done__func2 +; DATALEN is in [16, 32, 48, 64, 80, 96, 112]. - - - +; Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +; is the number of blocks that remain. lea rsi,[128+rdi] sub rsi,r8 - +; Start collecting the unreduced GHASH intermediate value LO, MI, HI. vpxor xmm5,xmm5,xmm5 vpxor xmm6,xmm6,xmm6 vpxor xmm7,xmm7,xmm7 @@ -1266,7 +1266,7 @@ cmp r8,64 jb NEAR $L$lessthan64bytes__func2 - +; DATALEN is in [64, 80, 96, 112]. Encrypt two vectors of counter blocks. vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,YMMWORD[$L$inc_2blocks] vpshufb ymm13,ymm11,ymm0 @@ -1284,7 +1284,7 @@ vaesenclast ymm12,ymm12,ymm10 vaesenclast ymm13,ymm13,ymm10 - +; XOR the data with the two vectors of keystream blocks. vmovdqu ymm2,YMMWORD[rcx] vmovdqu ymm3,YMMWORD[32+rcx] vpxor ymm12,ymm12,ymm2 @@ -1292,7 +1292,7 @@ vmovdqu YMMWORD[rdx],ymm12 vmovdqu YMMWORD[32+rdx],ymm13 - +; Update GHASH with two vectors of ciphertext blocks, without reducing. vpshufb ymm12,ymm2,ymm0 vpshufb ymm13,ymm3,ymm0 vpxor ymm12,ymm12,ymm1 @@ -1320,7 +1320,7 @@ vpxor xmm1,xmm1,xmm1 - +; DATALEN is in [16, 32, 48]. Encrypt two last vectors of counter blocks. $L$lessthan64bytes__func2: vpshufb ymm12,ymm11,ymm0 vpaddd ymm11,ymm11,YMMWORD[$L$inc_2blocks] @@ -1338,8 +1338,8 @@ vaesenclast ymm12,ymm12,ymm10 vaesenclast ymm13,ymm13,ymm10 - - +; XOR the remaining data with the keystream blocks, and update GHASH with +; the remaining ciphertext blocks without reducing. cmp r8,32 jb NEAR $L$xor_one_block__func2 @@ -1396,7 +1396,7 @@ vpxor ymm7,ymm7,ymm4 $L$reduce__func2: - +; Finally, do the GHASH reduction. vbroadcasti128 ymm2,XMMWORD[$L$gfpoly] vpclmulqdq ymm3,ymm2,ymm5,0x01 vpshufd ymm5,ymm5,0x4e @@ -1410,7 +1410,7 @@ vpxor xmm1,xmm1,xmm7 $L$done__func2: - +; Store the updated GHASH accumulator back to memory. vpshufb xmm1,xmm1,xmm0 vmovdqu XMMWORD[r12],xmm1 @@ -1459,7 +1459,7 @@ section .xdata rdata align=8 ALIGN 4 $L$SEH_info_gcm_init_vpclmulqdq_avx2_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_init_vpclmulqdq_avx2_4-$L$SEH_begin_gcm_init_vpclmulqdq_avx2_1 DB 3 DB 0 @@ -1471,7 +1471,7 @@ DW 0 $L$SEH_info_gcm_gmult_vpclmulqdq_avx2_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_gmult_vpclmulqdq_avx2_4-$L$SEH_begin_gcm_gmult_vpclmulqdq_avx2_1 DB 3 DB 0 @@ -1483,7 +1483,7 @@ DW 0 $L$SEH_info_gcm_ghash_vpclmulqdq_avx2_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_ghash_vpclmulqdq_avx2_7-$L$SEH_begin_gcm_ghash_vpclmulqdq_avx2_1 DB 9 DB 0 @@ -1504,7 +1504,7 @@ DW 0 $L$SEH_info_aes_gcm_enc_update_vaes_avx2_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aes_gcm_enc_update_vaes_avx2_16-$L$SEH_begin_aes_gcm_enc_update_vaes_avx2_1 DB 25 DB 0 @@ -1550,7 +1550,7 @@ DW 0 $L$SEH_info_aes_gcm_dec_update_vaes_avx2_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aes_gcm_dec_update_vaes_avx2_16-$L$SEH_begin_aes_gcm_dec_update_vaes_avx2_1 DB 25 DB 0
diff --git a/gen/bcm/aes-gcm-avx512-x86_64-apple.S b/gen/bcm/aes-gcm-avx512-x86_64-apple.S index 5d0b391..3e66ec6 100644 --- a/gen/bcm/aes-gcm-avx512-x86_64-apple.S +++ b/gen/bcm/aes-gcm-avx512-x86_64-apple.S
@@ -9,32 +9,32 @@ .p2align 6 aes_gcm_avx512_constants: - +// A shuffle mask that reflects the bytes of 16-byte blocks L$bswap_mask: .quad 0x08090a0b0c0d0e0f, 0x0001020304050607 - - - - - - - +// This is the GHASH reducing polynomial without its constant term, i.e. +// x^128 + x^7 + x^2 + x, represented using the backwards mapping +// between bits and polynomial coefficients. +// +// Alternatively, it can be interpreted as the naturally-ordered +// representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the +// "reversed" GHASH reducing polynomial without its x^128 term. L$gfpoly: .quad 1, 0xc200000000000000 - +// Same as above, but with the (1 << 64) bit set. L$gfpoly_and_internal_carrybit: .quad 1, 0xc200000000000001 - +// Values needed to prepare the initial vector of counter blocks. L$ctr_pattern: .quad 0, 0 .quad 1, 0 .quad 2, 0 .quad 3, 0 - +// The number of AES blocks per vector, as a 128-bit value. L$inc_4blocks: .quad 4, 0 @@ -47,100 +47,100 @@ _CET_ENDBR - +// Get pointer to lowest set of key powers (located at end of array). leaq 256-64(%rdi),%r8 - - +// Load the byte-reflected hash subkey. BoringSSL provides it in +// byte-reflected form except the two halves are in the wrong order. vpshufd $0x4e,(%rsi),%xmm3 - - - - - - - - - - - - - - - +// Finish preprocessing the first key power, H^1. Since this GHASH +// implementation operates directly on values with the backwards bit +// order specified by the GCM standard, it's necessary to preprocess the +// raw key as follows. First, reflect its bytes. Second, multiply it +// by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards +// interpretation of polynomial coefficients), which can also be +// interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121 +// + 1 using the alternative, natural interpretation of polynomial +// coefficients. For details, see the comment above _ghash_mul. +// +// Either way, for the multiplication the concrete operation performed +// is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2 +// << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit +// wide shift instruction, so instead double each of the two 64-bit +// halves and incorporate the internal carry bit into the value XOR'd. vpshufd $0xd3,%xmm3,%xmm0 vpsrad $31,%xmm0,%xmm0 vpaddq %xmm3,%xmm3,%xmm3 - +// H_CUR_XMM ^= TMP0_XMM & gfpoly_and_internal_carrybit vpternlogd $0x78,L$gfpoly_and_internal_carrybit(%rip),%xmm0,%xmm3 - +// Load the gfpoly constant. vbroadcasti32x4 L$gfpoly(%rip),%zmm5 +// Square H^1 to get H^2. +// +// Note that as with H^1, all higher key powers also need an extra +// factor of x^-1 (or x using the natural interpretation). Nothing +// special needs to be done to make this happen, though: H^1 * H^1 would +// end up with two factors of x^-1, but the multiplication consumes one. +// So the product H^2 ends up with the desired one factor of x^-1. + vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 // LO = a_L * a_L + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm4 // HI = a_H * a_H + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm1 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm0,%xmm0 // Swap halves of LO + vpxor %xmm0,%xmm1,%xmm1 // Fold LO into MI + vpclmulqdq $0x01,%xmm1,%xmm5,%xmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of MI + vpternlogd $0x96,%xmm0,%xmm1,%xmm4 // Fold MI into HI - - - - - - vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x11,%xmm3,%xmm3,%xmm4 - vpclmulqdq $0x01,%xmm0,%xmm5,%xmm1 - vpshufd $0x4e,%xmm0,%xmm0 - vpxor %xmm0,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm5,%xmm0 - vpshufd $0x4e,%xmm1,%xmm1 - vpternlogd $0x96,%xmm0,%xmm1,%xmm4 - - - +// Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2]. vinserti128 $1,%xmm3,%ymm4,%ymm3 vinserti128 $1,%xmm4,%ymm4,%ymm4 - - vpclmulqdq $0x00,%ymm4,%ymm3,%ymm0 - vpclmulqdq $0x01,%ymm4,%ymm3,%ymm1 - vpclmulqdq $0x10,%ymm4,%ymm3,%ymm2 - vpxord %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpternlogd $0x96,%ymm2,%ymm0,%ymm1 - vpclmulqdq $0x11,%ymm4,%ymm3,%ymm4 - vpclmulqdq $0x01,%ymm1,%ymm5,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpternlogd $0x96,%ymm0,%ymm1,%ymm4 +// Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4]. + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm4,%ymm3,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm4,%ymm3,%ymm2 // MI_1 = a_H * b_L + vpxord %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpternlogd $0x96,%ymm2,%ymm0,%ymm1 // Fold LO into MI + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm4 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm5,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpternlogd $0x96,%ymm0,%ymm1,%ymm4 // Fold MI into HI vinserti64x4 $1,%ymm3,%zmm4,%zmm3 vshufi64x2 $0,%zmm4,%zmm4,%zmm4 - +// Store the lowest set of key powers. vmovdqu8 %zmm3,(%r8) - - - +// Compute and store the remaining key powers. +// Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by +// [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)]. movl $3,%eax L$precompute_next: subq $64,%r8 - vpclmulqdq $0x00,%zmm4,%zmm3,%zmm0 - vpclmulqdq $0x01,%zmm4,%zmm3,%zmm1 - vpclmulqdq $0x10,%zmm4,%zmm3,%zmm2 - vpxord %zmm2,%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm0,%zmm5,%zmm2 - vpshufd $0x4e,%zmm0,%zmm0 - vpternlogd $0x96,%zmm2,%zmm0,%zmm1 - vpclmulqdq $0x11,%zmm4,%zmm3,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm5,%zmm0 - vpshufd $0x4e,%zmm1,%zmm1 - vpternlogd $0x96,%zmm0,%zmm1,%zmm3 + vpclmulqdq $0x00,%zmm4,%zmm3,%zmm0 // LO = a_L * b_L + vpclmulqdq $0x01,%zmm4,%zmm3,%zmm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%zmm4,%zmm3,%zmm2 // MI_1 = a_H * b_L + vpxord %zmm2,%zmm1,%zmm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%zmm0,%zmm5,%zmm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm0,%zmm0 // Swap halves of LO + vpternlogd $0x96,%zmm2,%zmm0,%zmm1 // Fold LO into MI + vpclmulqdq $0x11,%zmm4,%zmm3,%zmm3 // HI = a_H * b_H + vpclmulqdq $0x01,%zmm1,%zmm5,%zmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm1,%zmm1 // Swap halves of MI + vpternlogd $0x96,%zmm0,%zmm1,%zmm3 // Fold MI into HI vmovdqu8 %zmm3,(%r8) decl %eax jnz L$precompute_next - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. ret @@ -163,23 +163,23 @@ vmovdqu L$gfpoly(%rip),%xmm3 vpshufb %xmm1,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 - vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 - vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 - vpxord %xmm6,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 - vpshufd $0x4e,%xmm4,%xmm4 - vpternlogd $0x96,%xmm6,%xmm4,%xmm5 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 - vpshufd $0x4e,%xmm5,%xmm5 - vpternlogd $0x96,%xmm4,%xmm5,%xmm0 + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 // MI_1 = a_H * b_L + vpxord %xmm6,%xmm5,%xmm5 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm4,%xmm4 // Swap halves of LO + vpternlogd $0x96,%xmm6,%xmm4,%xmm5 // Fold LO into MI + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm5,%xmm5 // Swap halves of MI + vpternlogd $0x96,%xmm4,%xmm5,%xmm0 // Fold MI into HI vpshufb %xmm1,%xmm0,%xmm0 vmovdqu %xmm0,(%rdi) - +// No need for vzeroupper, since only xmm registers were used. ret @@ -196,37 +196,37 @@ - - - +// Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, +// usually only 128-bit vectors will be used. So as an optimization, don't +// broadcast these constants to all 128-bit lanes quite yet. vmovdqu L$bswap_mask(%rip),%xmm4 vmovdqu L$gfpoly(%rip),%xmm10 - +// Load the GHASH accumulator. vmovdqu (%rdi),%xmm5 vpshufb %xmm4,%xmm5,%xmm5 - +// Optimize for AADLEN < 64 by checking for AADLEN < 64 before AADLEN < 256. cmpq $64,%rcx jb L$aad_blockbyblock - - +// AADLEN >= 64, so we'll operate on full vectors. Broadcast bswap_mask and +// gfpoly to all 128-bit lanes. vshufi64x2 $0,%zmm4,%zmm4,%zmm4 vshufi64x2 $0,%zmm10,%zmm10,%zmm10 - +// Load the lowest set of key powers. vmovdqu8 256-64(%rsi),%zmm9 cmpq $256,%rcx jb L$aad_loop_1x - +// AADLEN >= 256. Load the higher key powers. vmovdqu8 256-256(%rsi),%zmm6 vmovdqu8 256-192(%rsi),%zmm7 vmovdqu8 256-128(%rsi),%zmm8 - +// Update GHASH with 256 bytes of AAD at a time. L$aad_loop_4x: vmovdqu8 0(%rdx),%zmm0 vmovdqu8 64(%rdx),%zmm1 @@ -237,36 +237,36 @@ vpshufb %zmm4,%zmm1,%zmm1 vpshufb %zmm4,%zmm2,%zmm2 vpshufb %zmm4,%zmm3,%zmm3 - vpclmulqdq $0x00,%zmm6,%zmm0,%zmm5 - vpclmulqdq $0x00,%zmm7,%zmm1,%zmm11 - vpclmulqdq $0x00,%zmm8,%zmm2,%zmm12 - vpxord %zmm11,%zmm5,%zmm5 - vpclmulqdq $0x00,%zmm9,%zmm3,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm5 - vpclmulqdq $0x01,%zmm6,%zmm0,%zmm11 - vpclmulqdq $0x01,%zmm7,%zmm1,%zmm12 - vpclmulqdq $0x01,%zmm8,%zmm2,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm11 - vpclmulqdq $0x01,%zmm9,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm6,%zmm0,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm11 - vpclmulqdq $0x10,%zmm7,%zmm1,%zmm12 - vpclmulqdq $0x10,%zmm8,%zmm2,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm11 - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm13 - vpclmulqdq $0x10,%zmm9,%zmm3,%zmm12 - vpxord %zmm12,%zmm11,%zmm11 - vpshufd $0x4e,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm6,%zmm0,%zmm0 - vpclmulqdq $0x11,%zmm7,%zmm1,%zmm1 - vpclmulqdq $0x11,%zmm8,%zmm2,%zmm2 - vpternlogd $0x96,%zmm13,%zmm5,%zmm11 - vpclmulqdq $0x11,%zmm9,%zmm3,%zmm3 - vpternlogd $0x96,%zmm2,%zmm1,%zmm0 - vpclmulqdq $0x01,%zmm11,%zmm10,%zmm12 - vpxord %zmm3,%zmm0,%zmm5 - vpshufd $0x4e,%zmm11,%zmm11 - vpternlogd $0x96,%zmm12,%zmm11,%zmm5 + vpclmulqdq $0x00,%zmm6,%zmm0,%zmm5 // LO_0 + vpclmulqdq $0x00,%zmm7,%zmm1,%zmm11 // LO_1 + vpclmulqdq $0x00,%zmm8,%zmm2,%zmm12 // LO_2 + vpxord %zmm11,%zmm5,%zmm5 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm9,%zmm3,%zmm13 // LO_3 + vpternlogd $0x96,%zmm13,%zmm12,%zmm5 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm6,%zmm0,%zmm11 // MI_0 + vpclmulqdq $0x01,%zmm7,%zmm1,%zmm12 // MI_1 + vpclmulqdq $0x01,%zmm8,%zmm2,%zmm13 // MI_2 + vpternlogd $0x96,%zmm13,%zmm12,%zmm11 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm9,%zmm3,%zmm12 // MI_3 + vpclmulqdq $0x10,%zmm6,%zmm0,%zmm13 // MI_4 + vpternlogd $0x96,%zmm13,%zmm12,%zmm11 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm7,%zmm1,%zmm12 // MI_5 + vpclmulqdq $0x10,%zmm8,%zmm2,%zmm13 // MI_6 + vpternlogd $0x96,%zmm13,%zmm12,%zmm11 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm5,%zmm10,%zmm13 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm9,%zmm3,%zmm12 // MI_7 + vpxord %zmm12,%zmm11,%zmm11 // MI = sum(MI_{7,6,5,4,3,2,1,0}) + vpshufd $0x4e,%zmm5,%zmm5 // Swap halves of LO + vpclmulqdq $0x11,%zmm6,%zmm0,%zmm0 // HI_0 + vpclmulqdq $0x11,%zmm7,%zmm1,%zmm1 // HI_1 + vpclmulqdq $0x11,%zmm8,%zmm2,%zmm2 // HI_2 + vpternlogd $0x96,%zmm13,%zmm5,%zmm11 // Fold LO into MI + vpclmulqdq $0x11,%zmm9,%zmm3,%zmm3 // HI_3 + vpternlogd $0x96,%zmm2,%zmm1,%zmm0 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm11,%zmm10,%zmm12 // MI_L*(x^63 + x^62 + x^57) + vpxord %zmm3,%zmm0,%zmm5 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm11,%zmm11 // Swap halves of MI + vpternlogd $0x96,%zmm12,%zmm11,%zmm5 // Fold MI into HI vextracti32x4 $1,%zmm5,%xmm0 vextracti32x4 $2,%zmm5,%xmm1 vextracti32x4 $3,%zmm5,%xmm2 @@ -278,24 +278,24 @@ cmpq $256,%rcx jae L$aad_loop_4x - +// Update GHASH with 64 bytes of AAD at a time. cmpq $64,%rcx jb L$aad_large_done L$aad_loop_1x: vmovdqu8 (%rdx),%zmm0 vpshufb %zmm4,%zmm0,%zmm0 vpxord %zmm0,%zmm5,%zmm5 - vpclmulqdq $0x00,%zmm9,%zmm5,%zmm0 - vpclmulqdq $0x01,%zmm9,%zmm5,%zmm1 - vpclmulqdq $0x10,%zmm9,%zmm5,%zmm2 - vpxord %zmm2,%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm0,%zmm10,%zmm2 - vpshufd $0x4e,%zmm0,%zmm0 - vpternlogd $0x96,%zmm2,%zmm0,%zmm1 - vpclmulqdq $0x11,%zmm9,%zmm5,%zmm5 - vpclmulqdq $0x01,%zmm1,%zmm10,%zmm0 - vpshufd $0x4e,%zmm1,%zmm1 - vpternlogd $0x96,%zmm0,%zmm1,%zmm5 + vpclmulqdq $0x00,%zmm9,%zmm5,%zmm0 // LO = a_L * b_L + vpclmulqdq $0x01,%zmm9,%zmm5,%zmm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%zmm9,%zmm5,%zmm2 // MI_1 = a_H * b_L + vpxord %zmm2,%zmm1,%zmm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%zmm0,%zmm10,%zmm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm0,%zmm0 // Swap halves of LO + vpternlogd $0x96,%zmm2,%zmm0,%zmm1 // Fold LO into MI + vpclmulqdq $0x11,%zmm9,%zmm5,%zmm5 // HI = a_H * b_H + vpclmulqdq $0x01,%zmm1,%zmm10,%zmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm1,%zmm1 // Swap halves of MI + vpternlogd $0x96,%zmm0,%zmm1,%zmm5 // Fold MI into HI vextracti32x4 $1,%zmm5,%xmm0 vextracti32x4 $2,%zmm5,%xmm1 @@ -310,7 +310,7 @@ L$aad_large_done: - +// GHASH the remaining data 16 bytes at a time, using xmm registers only. L$aad_blockbyblock: testq %rcx,%rcx jz L$aad_done @@ -319,28 +319,28 @@ vmovdqu (%rdx),%xmm0 vpshufb %xmm4,%xmm0,%xmm0 vpxor %xmm0,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm9,%xmm5,%xmm0 - vpclmulqdq $0x01,%xmm9,%xmm5,%xmm1 - vpclmulqdq $0x10,%xmm9,%xmm5,%xmm2 - vpxord %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm10,%xmm2 - vpshufd $0x4e,%xmm0,%xmm0 - vpternlogd $0x96,%xmm2,%xmm0,%xmm1 - vpclmulqdq $0x11,%xmm9,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm1,%xmm10,%xmm0 - vpshufd $0x4e,%xmm1,%xmm1 - vpternlogd $0x96,%xmm0,%xmm1,%xmm5 + vpclmulqdq $0x00,%xmm9,%xmm5,%xmm0 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm9,%xmm5,%xmm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm9,%xmm5,%xmm2 // MI_1 = a_H * b_L + vpxord %xmm2,%xmm1,%xmm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm0,%xmm10,%xmm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm0,%xmm0 // Swap halves of LO + vpternlogd $0x96,%xmm2,%xmm0,%xmm1 // Fold LO into MI + vpclmulqdq $0x11,%xmm9,%xmm5,%xmm5 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm1,%xmm10,%xmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of MI + vpternlogd $0x96,%xmm0,%xmm1,%xmm5 // Fold MI into HI addq $16,%rdx subq $16,%rcx jnz L$aad_loop_blockbyblock L$aad_done: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm4,%xmm5,%xmm5 vmovdqu %xmm5,(%rdi) - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. ret @@ -357,54 +357,54 @@ pushq %r12 - movq 16(%rsp),%r12 + movq 16(%rsp),%r12 // arg7 #ifdef BORINGSSL_DISPATCH_TEST movb $1,_BORINGSSL_function_hit+7(%rip) #endif - +// Load some constants. vbroadcasti32x4 L$bswap_mask(%rip),%zmm8 vbroadcasti32x4 L$gfpoly(%rip),%zmm31 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm10 vpshufb %xmm8,%xmm10,%xmm10 vbroadcasti32x4 (%r8),%zmm12 vpshufb %zmm8,%zmm12,%zmm12 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti32x4 (%rcx),%zmm13 vbroadcasti32x4 (%r11),%zmm14 - +// Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd L$ctr_pattern(%rip),%zmm12,%zmm12 - +// Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 L$inc_4blocks(%rip),%zmm11 - - +// If there are at least 256 bytes of data, then continue into the loop +// that processes 256 bytes of data at a time. Otherwise skip it. cmpq $256,%rdx jb L$crypt_loop_4x_done__func1 - +// Load powers of the hash key. vmovdqu8 256-256(%r9),%zmm27 vmovdqu8 256-192(%r9),%zmm28 vmovdqu8 256-128(%r9),%zmm29 vmovdqu8 256-64(%r9),%zmm30 - - - - +// Encrypt the first 4 vectors of plaintext blocks. Leave the resulting +// ciphertext in GHASHDATA[0-3] for GHASH. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpshufb %zmm8,%zmm12,%zmm1 @@ -414,7 +414,7 @@ vpshufb %zmm8,%zmm12,%zmm3 vpaddd %zmm11,%zmm12,%zmm12 - +// AES "round zero": XOR in the zero-th round key. vpxord %zmm13,%zmm0,%zmm0 vpxord %zmm13,%zmm1,%zmm1 vpxord %zmm13,%zmm2,%zmm2 @@ -449,7 +449,7 @@ subq $256,%rdx cmpq $256,%rdx jb L$ghash_last_ciphertext_4x__func1 - +// Cache as many additional AES round keys as possible. vbroadcasti32x4 -144(%r11),%zmm15 vbroadcasti32x4 -128(%r11),%zmm16 vbroadcasti32x4 -112(%r11),%zmm17 @@ -461,9 +461,9 @@ vbroadcasti32x4 -16(%r11),%zmm23 L$crypt_loop_4x__func1: - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpshufb %zmm8,%zmm12,%zmm1 @@ -473,7 +473,7 @@ vpshufb %zmm8,%zmm12,%zmm3 vpaddd %zmm11,%zmm12,%zmm12 - +// AES "round zero": XOR in the zero-th round key. vpxord %zmm13,%zmm0,%zmm0 vpxord %zmm13,%zmm1,%zmm1 vpxord %zmm13,%zmm2,%zmm2 @@ -482,7 +482,7 @@ cmpl $24,%r10d jl L$aes128__func1 je L$aes192__func1 - +// AES-256 vbroadcasti32x4 -208(%r11),%zmm9 vaesenc %zmm9,%zmm0,%zmm0 vaesenc %zmm9,%zmm1,%zmm1 @@ -510,17 +510,17 @@ L$aes128__func1: - - - +// Prefetch the source data 512 bytes ahead into the L1 data cache, to +// improve performance when the hardware prefetcher is disabled. Assumes the +// L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 512+0(%rdi) prefetcht0 512+64(%rdi) prefetcht0 512+128(%rdi) prefetcht0 512+192(%rdi) - - - +// Finish the AES encryption of the counter blocks in AESDATA[0-3], +// interleaved with the GHASH update of the ciphertext blocks in +// GHASHDATA[0-3]. vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 @@ -532,78 +532,78 @@ vaesenc %zmm15,%zmm3,%zmm3 vpshufb %zmm8,%zmm7,%zmm7 - vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 - vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 - vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 + vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 // LO_0 + vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 // LO_1 + vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 // LO_2 vaesenc %zmm16,%zmm0,%zmm0 vaesenc %zmm16,%zmm1,%zmm1 vaesenc %zmm16,%zmm2,%zmm2 vaesenc %zmm16,%zmm3,%zmm3 - vpxord %zmm24,%zmm10,%zmm10 - vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm10 - vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 + vpxord %zmm24,%zmm10,%zmm10 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 // LO_3 + vpternlogd $0x96,%zmm26,%zmm25,%zmm10 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 // MI_0 vaesenc %zmm17,%zmm0,%zmm0 vaesenc %zmm17,%zmm1,%zmm1 vaesenc %zmm17,%zmm2,%zmm2 vaesenc %zmm17,%zmm3,%zmm3 - vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 + vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 // MI_1 + vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 // MI_2 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 // MI_3 vaesenc %zmm18,%zmm0,%zmm0 vaesenc %zmm18,%zmm1,%zmm1 vaesenc %zmm18,%zmm2,%zmm2 vaesenc %zmm18,%zmm3,%zmm3 - vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 + vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 // MI_4 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 // MI_5 + vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 // MI_6 vaesenc %zmm19,%zmm0,%zmm0 vaesenc %zmm19,%zmm1,%zmm1 vaesenc %zmm19,%zmm2,%zmm2 vaesenc %zmm19,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 - vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 - vpxord %zmm25,%zmm24,%zmm24 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 // MI_7 + vpxord %zmm25,%zmm24,%zmm24 // MI = sum(MI_{7,6,5,4,3,2,1,0}) vaesenc %zmm20,%zmm0,%zmm0 vaesenc %zmm20,%zmm1,%zmm1 vaesenc %zmm20,%zmm2,%zmm2 vaesenc %zmm20,%zmm3,%zmm3 - vpshufd $0x4e,%zmm10,%zmm10 - vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 - vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 + vpshufd $0x4e,%zmm10,%zmm10 // Swap halves of LO + vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 // HI_0 + vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 // HI_1 + vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 // HI_2 vaesenc %zmm21,%zmm0,%zmm0 vaesenc %zmm21,%zmm1,%zmm1 vaesenc %zmm21,%zmm2,%zmm2 vaesenc %zmm21,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm10,%zmm24 - vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 - vpternlogd $0x96,%zmm6,%zmm5,%zmm4 - vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 + vpternlogd $0x96,%zmm26,%zmm10,%zmm24 // Fold LO into MI + vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 // HI_3 + vpternlogd $0x96,%zmm6,%zmm5,%zmm4 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 // MI_L*(x^63 + x^62 + x^57) vaesenc %zmm22,%zmm0,%zmm0 vaesenc %zmm22,%zmm1,%zmm1 vaesenc %zmm22,%zmm2,%zmm2 vaesenc %zmm22,%zmm3,%zmm3 - vpxord %zmm7,%zmm4,%zmm10 - vpshufd $0x4e,%zmm24,%zmm24 - vpternlogd $0x96,%zmm25,%zmm24,%zmm10 + vpxord %zmm7,%zmm4,%zmm10 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm24,%zmm24 // Swap halves of MI + vpternlogd $0x96,%zmm25,%zmm24,%zmm10 // Fold MI into HI vaesenc %zmm23,%zmm0,%zmm0 vaesenc %zmm23,%zmm1,%zmm1 @@ -641,36 +641,36 @@ vpshufb %zmm8,%zmm5,%zmm5 vpshufb %zmm8,%zmm6,%zmm6 vpshufb %zmm8,%zmm7,%zmm7 - vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 - vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 - vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 - vpxord %zmm24,%zmm10,%zmm10 - vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm10 - vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 - vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 - vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 - vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 - vpxord %zmm25,%zmm24,%zmm24 - vpshufd $0x4e,%zmm10,%zmm10 - vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 - vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 - vpternlogd $0x96,%zmm26,%zmm10,%zmm24 - vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 - vpternlogd $0x96,%zmm6,%zmm5,%zmm4 - vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 - vpxord %zmm7,%zmm4,%zmm10 - vpshufd $0x4e,%zmm24,%zmm24 - vpternlogd $0x96,%zmm25,%zmm24,%zmm10 + vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 // LO_0 + vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 // LO_1 + vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 // LO_2 + vpxord %zmm24,%zmm10,%zmm10 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 // LO_3 + vpternlogd $0x96,%zmm26,%zmm25,%zmm10 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 // MI_0 + vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 // MI_1 + vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 // MI_2 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 // MI_3 + vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 // MI_4 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 // MI_5 + vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 // MI_6 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 // MI_7 + vpxord %zmm25,%zmm24,%zmm24 // MI = sum(MI_{7,6,5,4,3,2,1,0}) + vpshufd $0x4e,%zmm10,%zmm10 // Swap halves of LO + vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 // HI_0 + vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 // HI_1 + vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 // HI_2 + vpternlogd $0x96,%zmm26,%zmm10,%zmm24 // Fold LO into MI + vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 // HI_3 + vpternlogd $0x96,%zmm6,%zmm5,%zmm4 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 // MI_L*(x^63 + x^62 + x^57) + vpxord %zmm7,%zmm4,%zmm10 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm24,%zmm24 // Swap halves of MI + vpternlogd $0x96,%zmm25,%zmm24,%zmm10 // Fold MI into HI vextracti32x4 $1,%zmm10,%xmm4 vextracti32x4 $2,%zmm10,%xmm5 vextracti32x4 $3,%zmm10,%xmm6 @@ -678,32 +678,32 @@ vpternlogd $0x96,%xmm5,%xmm6,%xmm10 L$crypt_loop_4x_done__func1: - +// Check whether any data remains. testq %rdx,%rdx jz L$done__func1 +// The data length isn't a multiple of 256 bytes. Process the remaining +// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. +// Going one vector at a time may seem inefficient compared to having +// separate code paths for each possible number of vectors remaining. +// However, using a loop keeps the code size down, and it performs +// surprising well; modern CPUs will start executing the next iteration +// before the previous one finishes and also predict the number of loop +// iterations. For a similar reason, we roll up the AES rounds. +// +// On the last iteration, the remaining length may be less than 64 bytes. +// Handle this using masking. +// +// Since there are enough key powers available for all remaining data, +// there is no need to do a GHASH reduction after each iteration. +// Instead, multiply each remaining block by its own key power, and only +// do a GHASH reduction at the very end. - - - - - - - - - - - - - - - - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. movq %rdx,%rax negq %rax - andq $-16,%rax + andq $-16,%rax // -round_up(DATALEN, 16) leaq 256(%r9,%rax,1),%r8 vpxor %xmm4,%xmm4,%xmm4 vpxor %xmm5,%xmm5,%xmm5 @@ -713,9 +713,9 @@ jb L$partial_vec__func1 L$crypt_loop_1x__func1: +// Process a full 64-byte vector. - - +// Encrypt a vector of counter blocks. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpxord %zmm13,%zmm0,%zmm0 @@ -728,19 +728,19 @@ jne L$vaesenc_loop_tail_full_vec__func1 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the vector of keystream blocks. vmovdqu8 (%rdi),%zmm1 vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi) - +// Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 (%r8),%zmm30 vpshufb %zmm8,%zmm0,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 @@ -758,10 +758,10 @@ jz L$reduce__func1 L$partial_vec__func1: +// Process a partial vector of length 1 <= DATALEN < 64. - - - +// Set the data mask %k1 to DATALEN 1's. +// Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. movq $-1,%rax bzhiq %rdx,%rax,%rax kmovq %rax,%k1 @@ -771,8 +771,8 @@ bzhiq %rdx,%rax,%rax kmovq %rax,%k2 - - +// Encrypt one last vector of counter blocks. This does not need to be +// masked. The counter does not need to be incremented here. vpshufb %zmm8,%zmm12,%zmm0 vpxord %zmm13,%zmm0,%zmm0 leaq 16(%rcx),%rax @@ -784,38 +784,38 @@ jne L$vaesenc_loop_tail_partialvec__func1 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the appropriate number of keystream bytes. vmovdqu8 (%rdi),%zmm1{%k1}{z} vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi){%k1} - - - - - - - - - - - - +// Update GHASH with the ciphertext block(s), without reducing. +// +// In the case of DATALEN < 64, the ciphertext is zero-padded to 64 +// bytes. (If decrypting, it's done by the above masked load. If +// encrypting, it's done by the below masked register-to-register move.) +// Note that if DATALEN <= 48, there will be additional padding beyond +// the padding of the last block specified by GHASH itself; i.e., there +// may be whole block(s) that get processed by the GHASH multiplication +// and reduction instructions but should not actually be included in the +// GHASH. However, any such blocks are all-zeroes, and the values that +// they're multiplied with are also all-zeroes. Therefore they just add +// 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 (%r8),%zmm30{%k2}{z} vmovdqu8 %zmm0,%zmm1{%k1}{z} vpshufb %zmm8,%zmm1,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 L$reduce__func1: - +// Finally, do the GHASH reduction. vpclmulqdq $0x01,%zmm4,%zmm31,%zmm0 vpshufd $0x4e,%zmm4,%zmm4 vpternlogd $0x96,%zmm0,%zmm4,%zmm5 @@ -831,11 +831,11 @@ L$done__func1: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm8,%xmm10,%xmm10 vmovdqu %xmm10,(%r12) - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. popq %r12 ret @@ -854,47 +854,47 @@ pushq %r12 - movq 16(%rsp),%r12 - + movq 16(%rsp),%r12 // arg7 +// Load some constants. vbroadcasti32x4 L$bswap_mask(%rip),%zmm8 vbroadcasti32x4 L$gfpoly(%rip),%zmm31 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm10 vpshufb %xmm8,%xmm10,%xmm10 vbroadcasti32x4 (%r8),%zmm12 vpshufb %zmm8,%zmm12,%zmm12 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti32x4 (%rcx),%zmm13 vbroadcasti32x4 (%r11),%zmm14 - +// Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd L$ctr_pattern(%rip),%zmm12,%zmm12 - +// Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 L$inc_4blocks(%rip),%zmm11 - - +// If there are at least 256 bytes of data, then continue into the loop +// that processes 256 bytes of data at a time. Otherwise skip it. cmpq $256,%rdx jb L$crypt_loop_4x_done__func2 - +// Load powers of the hash key. vmovdqu8 256-256(%r9),%zmm27 vmovdqu8 256-192(%r9),%zmm28 vmovdqu8 256-128(%r9),%zmm29 vmovdqu8 256-64(%r9),%zmm30 - +// Cache as many additional AES round keys as possible. vbroadcasti32x4 -144(%r11),%zmm15 vbroadcasti32x4 -128(%r11),%zmm16 vbroadcasti32x4 -112(%r11),%zmm17 @@ -910,9 +910,9 @@ vmovdqu8 64(%rdi),%zmm5 vmovdqu8 128(%rdi),%zmm6 vmovdqu8 192(%rdi),%zmm7 - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpshufb %zmm8,%zmm12,%zmm1 @@ -922,7 +922,7 @@ vpshufb %zmm8,%zmm12,%zmm3 vpaddd %zmm11,%zmm12,%zmm12 - +// AES "round zero": XOR in the zero-th round key. vpxord %zmm13,%zmm0,%zmm0 vpxord %zmm13,%zmm1,%zmm1 vpxord %zmm13,%zmm2,%zmm2 @@ -931,7 +931,7 @@ cmpl $24,%r10d jl L$aes128__func2 je L$aes192__func2 - +// AES-256 vbroadcasti32x4 -208(%r11),%zmm9 vaesenc %zmm9,%zmm0,%zmm0 vaesenc %zmm9,%zmm1,%zmm1 @@ -959,17 +959,17 @@ L$aes128__func2: - - - +// Prefetch the source data 512 bytes ahead into the L1 data cache, to +// improve performance when the hardware prefetcher is disabled. Assumes the +// L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 512+0(%rdi) prefetcht0 512+64(%rdi) prefetcht0 512+128(%rdi) prefetcht0 512+192(%rdi) - - - +// Finish the AES encryption of the counter blocks in AESDATA[0-3], +// interleaved with the GHASH update of the ciphertext blocks in +// GHASHDATA[0-3]. vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 @@ -981,78 +981,78 @@ vaesenc %zmm15,%zmm3,%zmm3 vpshufb %zmm8,%zmm7,%zmm7 - vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 - vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 - vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 + vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 // LO_0 + vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 // LO_1 + vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 // LO_2 vaesenc %zmm16,%zmm0,%zmm0 vaesenc %zmm16,%zmm1,%zmm1 vaesenc %zmm16,%zmm2,%zmm2 vaesenc %zmm16,%zmm3,%zmm3 - vpxord %zmm24,%zmm10,%zmm10 - vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm10 - vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 + vpxord %zmm24,%zmm10,%zmm10 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 // LO_3 + vpternlogd $0x96,%zmm26,%zmm25,%zmm10 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 // MI_0 vaesenc %zmm17,%zmm0,%zmm0 vaesenc %zmm17,%zmm1,%zmm1 vaesenc %zmm17,%zmm2,%zmm2 vaesenc %zmm17,%zmm3,%zmm3 - vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 + vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 // MI_1 + vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 // MI_2 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 // MI_3 vaesenc %zmm18,%zmm0,%zmm0 vaesenc %zmm18,%zmm1,%zmm1 vaesenc %zmm18,%zmm2,%zmm2 vaesenc %zmm18,%zmm3,%zmm3 - vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 + vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 // MI_4 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 // MI_5 + vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 // MI_6 vaesenc %zmm19,%zmm0,%zmm0 vaesenc %zmm19,%zmm1,%zmm1 vaesenc %zmm19,%zmm2,%zmm2 vaesenc %zmm19,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 - vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 - vpxord %zmm25,%zmm24,%zmm24 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 // MI_7 + vpxord %zmm25,%zmm24,%zmm24 // MI = sum(MI_{7,6,5,4,3,2,1,0}) vaesenc %zmm20,%zmm0,%zmm0 vaesenc %zmm20,%zmm1,%zmm1 vaesenc %zmm20,%zmm2,%zmm2 vaesenc %zmm20,%zmm3,%zmm3 - vpshufd $0x4e,%zmm10,%zmm10 - vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 - vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 + vpshufd $0x4e,%zmm10,%zmm10 // Swap halves of LO + vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 // HI_0 + vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 // HI_1 + vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 // HI_2 vaesenc %zmm21,%zmm0,%zmm0 vaesenc %zmm21,%zmm1,%zmm1 vaesenc %zmm21,%zmm2,%zmm2 vaesenc %zmm21,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm10,%zmm24 - vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 - vpternlogd $0x96,%zmm6,%zmm5,%zmm4 - vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 + vpternlogd $0x96,%zmm26,%zmm10,%zmm24 // Fold LO into MI + vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 // HI_3 + vpternlogd $0x96,%zmm6,%zmm5,%zmm4 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 // MI_L*(x^63 + x^62 + x^57) vaesenc %zmm22,%zmm0,%zmm0 vaesenc %zmm22,%zmm1,%zmm1 vaesenc %zmm22,%zmm2,%zmm2 vaesenc %zmm22,%zmm3,%zmm3 - vpxord %zmm7,%zmm4,%zmm10 - vpshufd $0x4e,%zmm24,%zmm24 - vpternlogd $0x96,%zmm25,%zmm24,%zmm10 + vpxord %zmm7,%zmm4,%zmm10 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm24,%zmm24 // Swap halves of MI + vpternlogd $0x96,%zmm25,%zmm24,%zmm10 // Fold MI into HI vaesenc %zmm23,%zmm0,%zmm0 vaesenc %zmm23,%zmm1,%zmm1 @@ -1085,32 +1085,32 @@ cmpq $256,%rdx jae L$crypt_loop_4x__func2 L$crypt_loop_4x_done__func2: - +// Check whether any data remains. testq %rdx,%rdx jz L$done__func2 +// The data length isn't a multiple of 256 bytes. Process the remaining +// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. +// Going one vector at a time may seem inefficient compared to having +// separate code paths for each possible number of vectors remaining. +// However, using a loop keeps the code size down, and it performs +// surprising well; modern CPUs will start executing the next iteration +// before the previous one finishes and also predict the number of loop +// iterations. For a similar reason, we roll up the AES rounds. +// +// On the last iteration, the remaining length may be less than 64 bytes. +// Handle this using masking. +// +// Since there are enough key powers available for all remaining data, +// there is no need to do a GHASH reduction after each iteration. +// Instead, multiply each remaining block by its own key power, and only +// do a GHASH reduction at the very end. - - - - - - - - - - - - - - - - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. movq %rdx,%rax negq %rax - andq $-16,%rax + andq $-16,%rax // -round_up(DATALEN, 16) leaq 256(%r9,%rax,1),%r8 vpxor %xmm4,%xmm4,%xmm4 vpxor %xmm5,%xmm5,%xmm5 @@ -1120,9 +1120,9 @@ jb L$partial_vec__func2 L$crypt_loop_1x__func2: +// Process a full 64-byte vector. - - +// Encrypt a vector of counter blocks. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpxord %zmm13,%zmm0,%zmm0 @@ -1135,19 +1135,19 @@ jne L$vaesenc_loop_tail_full_vec__func2 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the vector of keystream blocks. vmovdqu8 (%rdi),%zmm1 vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi) - +// Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 (%r8),%zmm30 vpshufb %zmm8,%zmm1,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 @@ -1165,10 +1165,10 @@ jz L$reduce__func2 L$partial_vec__func2: +// Process a partial vector of length 1 <= DATALEN < 64. - - - +// Set the data mask %k1 to DATALEN 1's. +// Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. movq $-1,%rax bzhiq %rdx,%rax,%rax kmovq %rax,%k1 @@ -1178,8 +1178,8 @@ bzhiq %rdx,%rax,%rax kmovq %rax,%k2 - - +// Encrypt one last vector of counter blocks. This does not need to be +// masked. The counter does not need to be incremented here. vpshufb %zmm8,%zmm12,%zmm0 vpxord %zmm13,%zmm0,%zmm0 leaq 16(%rcx),%rax @@ -1191,38 +1191,38 @@ jne L$vaesenc_loop_tail_partialvec__func2 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the appropriate number of keystream bytes. vmovdqu8 (%rdi),%zmm1{%k1}{z} vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi){%k1} - - - - - - - - - - - - +// Update GHASH with the ciphertext block(s), without reducing. +// +// In the case of DATALEN < 64, the ciphertext is zero-padded to 64 +// bytes. (If decrypting, it's done by the above masked load. If +// encrypting, it's done by the below masked register-to-register move.) +// Note that if DATALEN <= 48, there will be additional padding beyond +// the padding of the last block specified by GHASH itself; i.e., there +// may be whole block(s) that get processed by the GHASH multiplication +// and reduction instructions but should not actually be included in the +// GHASH. However, any such blocks are all-zeroes, and the values that +// they're multiplied with are also all-zeroes. Therefore they just add +// 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 (%r8),%zmm30{%k2}{z} vpshufb %zmm8,%zmm1,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 L$reduce__func2: - +// Finally, do the GHASH reduction. vpclmulqdq $0x01,%zmm4,%zmm31,%zmm0 vpshufd $0x4e,%zmm4,%zmm4 vpternlogd $0x96,%zmm0,%zmm4,%zmm5 @@ -1238,11 +1238,11 @@ L$done__func2: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm8,%xmm10,%xmm10 vmovdqu %xmm10,(%r12) - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. popq %r12 ret
diff --git a/gen/bcm/aes-gcm-avx512-x86_64-linux.S b/gen/bcm/aes-gcm-avx512-x86_64-linux.S index d4f1232..5bfc2bc 100644 --- a/gen/bcm/aes-gcm-avx512-x86_64-linux.S +++ b/gen/bcm/aes-gcm-avx512-x86_64-linux.S
@@ -9,32 +9,32 @@ .align 64 aes_gcm_avx512_constants: - +// A shuffle mask that reflects the bytes of 16-byte blocks .Lbswap_mask: .quad 0x08090a0b0c0d0e0f, 0x0001020304050607 - - - - - - - +// This is the GHASH reducing polynomial without its constant term, i.e. +// x^128 + x^7 + x^2 + x, represented using the backwards mapping +// between bits and polynomial coefficients. +// +// Alternatively, it can be interpreted as the naturally-ordered +// representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the +// "reversed" GHASH reducing polynomial without its x^128 term. .Lgfpoly: .quad 1, 0xc200000000000000 - +// Same as above, but with the (1 << 64) bit set. .Lgfpoly_and_internal_carrybit: .quad 1, 0xc200000000000001 - +// Values needed to prepare the initial vector of counter blocks. .Lctr_pattern: .quad 0, 0 .quad 1, 0 .quad 2, 0 .quad 3, 0 - +// The number of AES blocks per vector, as a 128-bit value. .Linc_4blocks: .quad 4, 0 @@ -47,100 +47,100 @@ .cfi_startproc _CET_ENDBR - +// Get pointer to lowest set of key powers (located at end of array). leaq 256-64(%rdi),%r8 - - +// Load the byte-reflected hash subkey. BoringSSL provides it in +// byte-reflected form except the two halves are in the wrong order. vpshufd $0x4e,(%rsi),%xmm3 - - - - - - - - - - - - - - - +// Finish preprocessing the first key power, H^1. Since this GHASH +// implementation operates directly on values with the backwards bit +// order specified by the GCM standard, it's necessary to preprocess the +// raw key as follows. First, reflect its bytes. Second, multiply it +// by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards +// interpretation of polynomial coefficients), which can also be +// interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121 +// + 1 using the alternative, natural interpretation of polynomial +// coefficients. For details, see the comment above _ghash_mul. +// +// Either way, for the multiplication the concrete operation performed +// is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2 +// << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit +// wide shift instruction, so instead double each of the two 64-bit +// halves and incorporate the internal carry bit into the value XOR'd. vpshufd $0xd3,%xmm3,%xmm0 vpsrad $31,%xmm0,%xmm0 vpaddq %xmm3,%xmm3,%xmm3 - +// H_CUR_XMM ^= TMP0_XMM & gfpoly_and_internal_carrybit vpternlogd $0x78,.Lgfpoly_and_internal_carrybit(%rip),%xmm0,%xmm3 - +// Load the gfpoly constant. vbroadcasti32x4 .Lgfpoly(%rip),%zmm5 +// Square H^1 to get H^2. +// +// Note that as with H^1, all higher key powers also need an extra +// factor of x^-1 (or x using the natural interpretation). Nothing +// special needs to be done to make this happen, though: H^1 * H^1 would +// end up with two factors of x^-1, but the multiplication consumes one. +// So the product H^2 ends up with the desired one factor of x^-1. + vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 // LO = a_L * a_L + vpclmulqdq $0x11,%xmm3,%xmm3,%xmm4 // HI = a_H * a_H + vpclmulqdq $0x01,%xmm0,%xmm5,%xmm1 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm0,%xmm0 // Swap halves of LO + vpxor %xmm0,%xmm1,%xmm1 // Fold LO into MI + vpclmulqdq $0x01,%xmm1,%xmm5,%xmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of MI + vpternlogd $0x96,%xmm0,%xmm1,%xmm4 // Fold MI into HI - - - - - - vpclmulqdq $0x00,%xmm3,%xmm3,%xmm0 - vpclmulqdq $0x11,%xmm3,%xmm3,%xmm4 - vpclmulqdq $0x01,%xmm0,%xmm5,%xmm1 - vpshufd $0x4e,%xmm0,%xmm0 - vpxor %xmm0,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm1,%xmm5,%xmm0 - vpshufd $0x4e,%xmm1,%xmm1 - vpternlogd $0x96,%xmm0,%xmm1,%xmm4 - - - +// Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2]. vinserti128 $1,%xmm3,%ymm4,%ymm3 vinserti128 $1,%xmm4,%ymm4,%ymm4 - - vpclmulqdq $0x00,%ymm4,%ymm3,%ymm0 - vpclmulqdq $0x01,%ymm4,%ymm3,%ymm1 - vpclmulqdq $0x10,%ymm4,%ymm3,%ymm2 - vpxord %ymm2,%ymm1,%ymm1 - vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 - vpshufd $0x4e,%ymm0,%ymm0 - vpternlogd $0x96,%ymm2,%ymm0,%ymm1 - vpclmulqdq $0x11,%ymm4,%ymm3,%ymm4 - vpclmulqdq $0x01,%ymm1,%ymm5,%ymm0 - vpshufd $0x4e,%ymm1,%ymm1 - vpternlogd $0x96,%ymm0,%ymm1,%ymm4 +// Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4]. + vpclmulqdq $0x00,%ymm4,%ymm3,%ymm0 // LO = a_L * b_L + vpclmulqdq $0x01,%ymm4,%ymm3,%ymm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%ymm4,%ymm3,%ymm2 // MI_1 = a_H * b_L + vpxord %ymm2,%ymm1,%ymm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%ymm0,%ymm5,%ymm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm0,%ymm0 // Swap halves of LO + vpternlogd $0x96,%ymm2,%ymm0,%ymm1 // Fold LO into MI + vpclmulqdq $0x11,%ymm4,%ymm3,%ymm4 // HI = a_H * b_H + vpclmulqdq $0x01,%ymm1,%ymm5,%ymm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%ymm1,%ymm1 // Swap halves of MI + vpternlogd $0x96,%ymm0,%ymm1,%ymm4 // Fold MI into HI vinserti64x4 $1,%ymm3,%zmm4,%zmm3 vshufi64x2 $0,%zmm4,%zmm4,%zmm4 - +// Store the lowest set of key powers. vmovdqu8 %zmm3,(%r8) - - - +// Compute and store the remaining key powers. +// Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by +// [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)]. movl $3,%eax .Lprecompute_next: subq $64,%r8 - vpclmulqdq $0x00,%zmm4,%zmm3,%zmm0 - vpclmulqdq $0x01,%zmm4,%zmm3,%zmm1 - vpclmulqdq $0x10,%zmm4,%zmm3,%zmm2 - vpxord %zmm2,%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm0,%zmm5,%zmm2 - vpshufd $0x4e,%zmm0,%zmm0 - vpternlogd $0x96,%zmm2,%zmm0,%zmm1 - vpclmulqdq $0x11,%zmm4,%zmm3,%zmm3 - vpclmulqdq $0x01,%zmm1,%zmm5,%zmm0 - vpshufd $0x4e,%zmm1,%zmm1 - vpternlogd $0x96,%zmm0,%zmm1,%zmm3 + vpclmulqdq $0x00,%zmm4,%zmm3,%zmm0 // LO = a_L * b_L + vpclmulqdq $0x01,%zmm4,%zmm3,%zmm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%zmm4,%zmm3,%zmm2 // MI_1 = a_H * b_L + vpxord %zmm2,%zmm1,%zmm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%zmm0,%zmm5,%zmm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm0,%zmm0 // Swap halves of LO + vpternlogd $0x96,%zmm2,%zmm0,%zmm1 // Fold LO into MI + vpclmulqdq $0x11,%zmm4,%zmm3,%zmm3 // HI = a_H * b_H + vpclmulqdq $0x01,%zmm1,%zmm5,%zmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm1,%zmm1 // Swap halves of MI + vpternlogd $0x96,%zmm0,%zmm1,%zmm3 // Fold MI into HI vmovdqu8 %zmm3,(%r8) decl %eax jnz .Lprecompute_next - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. ret .cfi_endproc @@ -162,23 +162,23 @@ vmovdqu .Lgfpoly(%rip),%xmm3 vpshufb %xmm1,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 - vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 - vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 - vpxord %xmm6,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 - vpshufd $0x4e,%xmm4,%xmm4 - vpternlogd $0x96,%xmm6,%xmm4,%xmm5 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 - vpshufd $0x4e,%xmm5,%xmm5 - vpternlogd $0x96,%xmm4,%xmm5,%xmm0 + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm4 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm2,%xmm0,%xmm5 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm2,%xmm0,%xmm6 // MI_1 = a_H * b_L + vpxord %xmm6,%xmm5,%xmm5 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm4,%xmm3,%xmm6 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm4,%xmm4 // Swap halves of LO + vpternlogd $0x96,%xmm6,%xmm4,%xmm5 // Fold LO into MI + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm0 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm5,%xmm3,%xmm4 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm5,%xmm5 // Swap halves of MI + vpternlogd $0x96,%xmm4,%xmm5,%xmm0 // Fold MI into HI vpshufb %xmm1,%xmm0,%xmm0 vmovdqu %xmm0,(%rdi) - +// No need for vzeroupper, since only xmm registers were used. ret .cfi_endproc @@ -194,37 +194,37 @@ - - - +// Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, +// usually only 128-bit vectors will be used. So as an optimization, don't +// broadcast these constants to all 128-bit lanes quite yet. vmovdqu .Lbswap_mask(%rip),%xmm4 vmovdqu .Lgfpoly(%rip),%xmm10 - +// Load the GHASH accumulator. vmovdqu (%rdi),%xmm5 vpshufb %xmm4,%xmm5,%xmm5 - +// Optimize for AADLEN < 64 by checking for AADLEN < 64 before AADLEN < 256. cmpq $64,%rcx jb .Laad_blockbyblock - - +// AADLEN >= 64, so we'll operate on full vectors. Broadcast bswap_mask and +// gfpoly to all 128-bit lanes. vshufi64x2 $0,%zmm4,%zmm4,%zmm4 vshufi64x2 $0,%zmm10,%zmm10,%zmm10 - +// Load the lowest set of key powers. vmovdqu8 256-64(%rsi),%zmm9 cmpq $256,%rcx jb .Laad_loop_1x - +// AADLEN >= 256. Load the higher key powers. vmovdqu8 256-256(%rsi),%zmm6 vmovdqu8 256-192(%rsi),%zmm7 vmovdqu8 256-128(%rsi),%zmm8 - +// Update GHASH with 256 bytes of AAD at a time. .Laad_loop_4x: vmovdqu8 0(%rdx),%zmm0 vmovdqu8 64(%rdx),%zmm1 @@ -235,36 +235,36 @@ vpshufb %zmm4,%zmm1,%zmm1 vpshufb %zmm4,%zmm2,%zmm2 vpshufb %zmm4,%zmm3,%zmm3 - vpclmulqdq $0x00,%zmm6,%zmm0,%zmm5 - vpclmulqdq $0x00,%zmm7,%zmm1,%zmm11 - vpclmulqdq $0x00,%zmm8,%zmm2,%zmm12 - vpxord %zmm11,%zmm5,%zmm5 - vpclmulqdq $0x00,%zmm9,%zmm3,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm5 - vpclmulqdq $0x01,%zmm6,%zmm0,%zmm11 - vpclmulqdq $0x01,%zmm7,%zmm1,%zmm12 - vpclmulqdq $0x01,%zmm8,%zmm2,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm11 - vpclmulqdq $0x01,%zmm9,%zmm3,%zmm12 - vpclmulqdq $0x10,%zmm6,%zmm0,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm11 - vpclmulqdq $0x10,%zmm7,%zmm1,%zmm12 - vpclmulqdq $0x10,%zmm8,%zmm2,%zmm13 - vpternlogd $0x96,%zmm13,%zmm12,%zmm11 - vpclmulqdq $0x01,%zmm5,%zmm10,%zmm13 - vpclmulqdq $0x10,%zmm9,%zmm3,%zmm12 - vpxord %zmm12,%zmm11,%zmm11 - vpshufd $0x4e,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm6,%zmm0,%zmm0 - vpclmulqdq $0x11,%zmm7,%zmm1,%zmm1 - vpclmulqdq $0x11,%zmm8,%zmm2,%zmm2 - vpternlogd $0x96,%zmm13,%zmm5,%zmm11 - vpclmulqdq $0x11,%zmm9,%zmm3,%zmm3 - vpternlogd $0x96,%zmm2,%zmm1,%zmm0 - vpclmulqdq $0x01,%zmm11,%zmm10,%zmm12 - vpxord %zmm3,%zmm0,%zmm5 - vpshufd $0x4e,%zmm11,%zmm11 - vpternlogd $0x96,%zmm12,%zmm11,%zmm5 + vpclmulqdq $0x00,%zmm6,%zmm0,%zmm5 // LO_0 + vpclmulqdq $0x00,%zmm7,%zmm1,%zmm11 // LO_1 + vpclmulqdq $0x00,%zmm8,%zmm2,%zmm12 // LO_2 + vpxord %zmm11,%zmm5,%zmm5 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm9,%zmm3,%zmm13 // LO_3 + vpternlogd $0x96,%zmm13,%zmm12,%zmm5 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm6,%zmm0,%zmm11 // MI_0 + vpclmulqdq $0x01,%zmm7,%zmm1,%zmm12 // MI_1 + vpclmulqdq $0x01,%zmm8,%zmm2,%zmm13 // MI_2 + vpternlogd $0x96,%zmm13,%zmm12,%zmm11 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm9,%zmm3,%zmm12 // MI_3 + vpclmulqdq $0x10,%zmm6,%zmm0,%zmm13 // MI_4 + vpternlogd $0x96,%zmm13,%zmm12,%zmm11 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm7,%zmm1,%zmm12 // MI_5 + vpclmulqdq $0x10,%zmm8,%zmm2,%zmm13 // MI_6 + vpternlogd $0x96,%zmm13,%zmm12,%zmm11 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm5,%zmm10,%zmm13 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm9,%zmm3,%zmm12 // MI_7 + vpxord %zmm12,%zmm11,%zmm11 // MI = sum(MI_{7,6,5,4,3,2,1,0}) + vpshufd $0x4e,%zmm5,%zmm5 // Swap halves of LO + vpclmulqdq $0x11,%zmm6,%zmm0,%zmm0 // HI_0 + vpclmulqdq $0x11,%zmm7,%zmm1,%zmm1 // HI_1 + vpclmulqdq $0x11,%zmm8,%zmm2,%zmm2 // HI_2 + vpternlogd $0x96,%zmm13,%zmm5,%zmm11 // Fold LO into MI + vpclmulqdq $0x11,%zmm9,%zmm3,%zmm3 // HI_3 + vpternlogd $0x96,%zmm2,%zmm1,%zmm0 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm11,%zmm10,%zmm12 // MI_L*(x^63 + x^62 + x^57) + vpxord %zmm3,%zmm0,%zmm5 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm11,%zmm11 // Swap halves of MI + vpternlogd $0x96,%zmm12,%zmm11,%zmm5 // Fold MI into HI vextracti32x4 $1,%zmm5,%xmm0 vextracti32x4 $2,%zmm5,%xmm1 vextracti32x4 $3,%zmm5,%xmm2 @@ -276,24 +276,24 @@ cmpq $256,%rcx jae .Laad_loop_4x - +// Update GHASH with 64 bytes of AAD at a time. cmpq $64,%rcx jb .Laad_large_done .Laad_loop_1x: vmovdqu8 (%rdx),%zmm0 vpshufb %zmm4,%zmm0,%zmm0 vpxord %zmm0,%zmm5,%zmm5 - vpclmulqdq $0x00,%zmm9,%zmm5,%zmm0 - vpclmulqdq $0x01,%zmm9,%zmm5,%zmm1 - vpclmulqdq $0x10,%zmm9,%zmm5,%zmm2 - vpxord %zmm2,%zmm1,%zmm1 - vpclmulqdq $0x01,%zmm0,%zmm10,%zmm2 - vpshufd $0x4e,%zmm0,%zmm0 - vpternlogd $0x96,%zmm2,%zmm0,%zmm1 - vpclmulqdq $0x11,%zmm9,%zmm5,%zmm5 - vpclmulqdq $0x01,%zmm1,%zmm10,%zmm0 - vpshufd $0x4e,%zmm1,%zmm1 - vpternlogd $0x96,%zmm0,%zmm1,%zmm5 + vpclmulqdq $0x00,%zmm9,%zmm5,%zmm0 // LO = a_L * b_L + vpclmulqdq $0x01,%zmm9,%zmm5,%zmm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%zmm9,%zmm5,%zmm2 // MI_1 = a_H * b_L + vpxord %zmm2,%zmm1,%zmm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%zmm0,%zmm10,%zmm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm0,%zmm0 // Swap halves of LO + vpternlogd $0x96,%zmm2,%zmm0,%zmm1 // Fold LO into MI + vpclmulqdq $0x11,%zmm9,%zmm5,%zmm5 // HI = a_H * b_H + vpclmulqdq $0x01,%zmm1,%zmm10,%zmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%zmm1,%zmm1 // Swap halves of MI + vpternlogd $0x96,%zmm0,%zmm1,%zmm5 // Fold MI into HI vextracti32x4 $1,%zmm5,%xmm0 vextracti32x4 $2,%zmm5,%xmm1 @@ -308,7 +308,7 @@ .Laad_large_done: - +// GHASH the remaining data 16 bytes at a time, using xmm registers only. .Laad_blockbyblock: testq %rcx,%rcx jz .Laad_done @@ -317,28 +317,28 @@ vmovdqu (%rdx),%xmm0 vpshufb %xmm4,%xmm0,%xmm0 vpxor %xmm0,%xmm5,%xmm5 - vpclmulqdq $0x00,%xmm9,%xmm5,%xmm0 - vpclmulqdq $0x01,%xmm9,%xmm5,%xmm1 - vpclmulqdq $0x10,%xmm9,%xmm5,%xmm2 - vpxord %xmm2,%xmm1,%xmm1 - vpclmulqdq $0x01,%xmm0,%xmm10,%xmm2 - vpshufd $0x4e,%xmm0,%xmm0 - vpternlogd $0x96,%xmm2,%xmm0,%xmm1 - vpclmulqdq $0x11,%xmm9,%xmm5,%xmm5 - vpclmulqdq $0x01,%xmm1,%xmm10,%xmm0 - vpshufd $0x4e,%xmm1,%xmm1 - vpternlogd $0x96,%xmm0,%xmm1,%xmm5 + vpclmulqdq $0x00,%xmm9,%xmm5,%xmm0 // LO = a_L * b_L + vpclmulqdq $0x01,%xmm9,%xmm5,%xmm1 // MI_0 = a_L * b_H + vpclmulqdq $0x10,%xmm9,%xmm5,%xmm2 // MI_1 = a_H * b_L + vpxord %xmm2,%xmm1,%xmm1 // MI = MI_0 + MI_1 + vpclmulqdq $0x01,%xmm0,%xmm10,%xmm2 // LO_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm0,%xmm0 // Swap halves of LO + vpternlogd $0x96,%xmm2,%xmm0,%xmm1 // Fold LO into MI + vpclmulqdq $0x11,%xmm9,%xmm5,%xmm5 // HI = a_H * b_H + vpclmulqdq $0x01,%xmm1,%xmm10,%xmm0 // MI_L*(x^63 + x^62 + x^57) + vpshufd $0x4e,%xmm1,%xmm1 // Swap halves of MI + vpternlogd $0x96,%xmm0,%xmm1,%xmm5 // Fold MI into HI addq $16,%rdx subq $16,%rcx jnz .Laad_loop_blockbyblock .Laad_done: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm4,%xmm5,%xmm5 vmovdqu %xmm5,(%rdi) - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. ret .cfi_endproc @@ -355,55 +355,55 @@ .cfi_adjust_cfa_offset 8 .cfi_offset %r12,-16 - movq 16(%rsp),%r12 + movq 16(%rsp),%r12 // arg7 #ifdef BORINGSSL_DISPATCH_TEST .extern BORINGSSL_function_hit .hidden BORINGSSL_function_hit movb $1,BORINGSSL_function_hit+7(%rip) #endif - +// Load some constants. vbroadcasti32x4 .Lbswap_mask(%rip),%zmm8 vbroadcasti32x4 .Lgfpoly(%rip),%zmm31 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm10 vpshufb %xmm8,%xmm10,%xmm10 vbroadcasti32x4 (%r8),%zmm12 vpshufb %zmm8,%zmm12,%zmm12 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti32x4 (%rcx),%zmm13 vbroadcasti32x4 (%r11),%zmm14 - +// Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd .Lctr_pattern(%rip),%zmm12,%zmm12 - +// Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 .Linc_4blocks(%rip),%zmm11 - - +// If there are at least 256 bytes of data, then continue into the loop +// that processes 256 bytes of data at a time. Otherwise skip it. cmpq $256,%rdx jb .Lcrypt_loop_4x_done__func1 - +// Load powers of the hash key. vmovdqu8 256-256(%r9),%zmm27 vmovdqu8 256-192(%r9),%zmm28 vmovdqu8 256-128(%r9),%zmm29 vmovdqu8 256-64(%r9),%zmm30 - - - - +// Encrypt the first 4 vectors of plaintext blocks. Leave the resulting +// ciphertext in GHASHDATA[0-3] for GHASH. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpshufb %zmm8,%zmm12,%zmm1 @@ -413,7 +413,7 @@ vpshufb %zmm8,%zmm12,%zmm3 vpaddd %zmm11,%zmm12,%zmm12 - +// AES "round zero": XOR in the zero-th round key. vpxord %zmm13,%zmm0,%zmm0 vpxord %zmm13,%zmm1,%zmm1 vpxord %zmm13,%zmm2,%zmm2 @@ -448,7 +448,7 @@ subq $256,%rdx cmpq $256,%rdx jb .Lghash_last_ciphertext_4x__func1 - +// Cache as many additional AES round keys as possible. vbroadcasti32x4 -144(%r11),%zmm15 vbroadcasti32x4 -128(%r11),%zmm16 vbroadcasti32x4 -112(%r11),%zmm17 @@ -460,9 +460,9 @@ vbroadcasti32x4 -16(%r11),%zmm23 .Lcrypt_loop_4x__func1: - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpshufb %zmm8,%zmm12,%zmm1 @@ -472,7 +472,7 @@ vpshufb %zmm8,%zmm12,%zmm3 vpaddd %zmm11,%zmm12,%zmm12 - +// AES "round zero": XOR in the zero-th round key. vpxord %zmm13,%zmm0,%zmm0 vpxord %zmm13,%zmm1,%zmm1 vpxord %zmm13,%zmm2,%zmm2 @@ -481,7 +481,7 @@ cmpl $24,%r10d jl .Laes128__func1 je .Laes192__func1 - +// AES-256 vbroadcasti32x4 -208(%r11),%zmm9 vaesenc %zmm9,%zmm0,%zmm0 vaesenc %zmm9,%zmm1,%zmm1 @@ -509,17 +509,17 @@ .Laes128__func1: - - - +// Prefetch the source data 512 bytes ahead into the L1 data cache, to +// improve performance when the hardware prefetcher is disabled. Assumes the +// L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 512+0(%rdi) prefetcht0 512+64(%rdi) prefetcht0 512+128(%rdi) prefetcht0 512+192(%rdi) - - - +// Finish the AES encryption of the counter blocks in AESDATA[0-3], +// interleaved with the GHASH update of the ciphertext blocks in +// GHASHDATA[0-3]. vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 @@ -531,78 +531,78 @@ vaesenc %zmm15,%zmm3,%zmm3 vpshufb %zmm8,%zmm7,%zmm7 - vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 - vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 - vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 + vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 // LO_0 + vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 // LO_1 + vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 // LO_2 vaesenc %zmm16,%zmm0,%zmm0 vaesenc %zmm16,%zmm1,%zmm1 vaesenc %zmm16,%zmm2,%zmm2 vaesenc %zmm16,%zmm3,%zmm3 - vpxord %zmm24,%zmm10,%zmm10 - vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm10 - vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 + vpxord %zmm24,%zmm10,%zmm10 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 // LO_3 + vpternlogd $0x96,%zmm26,%zmm25,%zmm10 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 // MI_0 vaesenc %zmm17,%zmm0,%zmm0 vaesenc %zmm17,%zmm1,%zmm1 vaesenc %zmm17,%zmm2,%zmm2 vaesenc %zmm17,%zmm3,%zmm3 - vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 + vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 // MI_1 + vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 // MI_2 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 // MI_3 vaesenc %zmm18,%zmm0,%zmm0 vaesenc %zmm18,%zmm1,%zmm1 vaesenc %zmm18,%zmm2,%zmm2 vaesenc %zmm18,%zmm3,%zmm3 - vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 + vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 // MI_4 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 // MI_5 + vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 // MI_6 vaesenc %zmm19,%zmm0,%zmm0 vaesenc %zmm19,%zmm1,%zmm1 vaesenc %zmm19,%zmm2,%zmm2 vaesenc %zmm19,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 - vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 - vpxord %zmm25,%zmm24,%zmm24 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 // MI_7 + vpxord %zmm25,%zmm24,%zmm24 // MI = sum(MI_{7,6,5,4,3,2,1,0}) vaesenc %zmm20,%zmm0,%zmm0 vaesenc %zmm20,%zmm1,%zmm1 vaesenc %zmm20,%zmm2,%zmm2 vaesenc %zmm20,%zmm3,%zmm3 - vpshufd $0x4e,%zmm10,%zmm10 - vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 - vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 + vpshufd $0x4e,%zmm10,%zmm10 // Swap halves of LO + vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 // HI_0 + vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 // HI_1 + vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 // HI_2 vaesenc %zmm21,%zmm0,%zmm0 vaesenc %zmm21,%zmm1,%zmm1 vaesenc %zmm21,%zmm2,%zmm2 vaesenc %zmm21,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm10,%zmm24 - vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 - vpternlogd $0x96,%zmm6,%zmm5,%zmm4 - vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 + vpternlogd $0x96,%zmm26,%zmm10,%zmm24 // Fold LO into MI + vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 // HI_3 + vpternlogd $0x96,%zmm6,%zmm5,%zmm4 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 // MI_L*(x^63 + x^62 + x^57) vaesenc %zmm22,%zmm0,%zmm0 vaesenc %zmm22,%zmm1,%zmm1 vaesenc %zmm22,%zmm2,%zmm2 vaesenc %zmm22,%zmm3,%zmm3 - vpxord %zmm7,%zmm4,%zmm10 - vpshufd $0x4e,%zmm24,%zmm24 - vpternlogd $0x96,%zmm25,%zmm24,%zmm10 + vpxord %zmm7,%zmm4,%zmm10 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm24,%zmm24 // Swap halves of MI + vpternlogd $0x96,%zmm25,%zmm24,%zmm10 // Fold MI into HI vaesenc %zmm23,%zmm0,%zmm0 vaesenc %zmm23,%zmm1,%zmm1 @@ -640,36 +640,36 @@ vpshufb %zmm8,%zmm5,%zmm5 vpshufb %zmm8,%zmm6,%zmm6 vpshufb %zmm8,%zmm7,%zmm7 - vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 - vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 - vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 - vpxord %zmm24,%zmm10,%zmm10 - vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm10 - vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 - vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 - vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 - vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 - vpxord %zmm25,%zmm24,%zmm24 - vpshufd $0x4e,%zmm10,%zmm10 - vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 - vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 - vpternlogd $0x96,%zmm26,%zmm10,%zmm24 - vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 - vpternlogd $0x96,%zmm6,%zmm5,%zmm4 - vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 - vpxord %zmm7,%zmm4,%zmm10 - vpshufd $0x4e,%zmm24,%zmm24 - vpternlogd $0x96,%zmm25,%zmm24,%zmm10 + vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 // LO_0 + vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 // LO_1 + vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 // LO_2 + vpxord %zmm24,%zmm10,%zmm10 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 // LO_3 + vpternlogd $0x96,%zmm26,%zmm25,%zmm10 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 // MI_0 + vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 // MI_1 + vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 // MI_2 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 // MI_3 + vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 // MI_4 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 // MI_5 + vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 // MI_6 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 // MI_7 + vpxord %zmm25,%zmm24,%zmm24 // MI = sum(MI_{7,6,5,4,3,2,1,0}) + vpshufd $0x4e,%zmm10,%zmm10 // Swap halves of LO + vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 // HI_0 + vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 // HI_1 + vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 // HI_2 + vpternlogd $0x96,%zmm26,%zmm10,%zmm24 // Fold LO into MI + vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 // HI_3 + vpternlogd $0x96,%zmm6,%zmm5,%zmm4 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 // MI_L*(x^63 + x^62 + x^57) + vpxord %zmm7,%zmm4,%zmm10 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm24,%zmm24 // Swap halves of MI + vpternlogd $0x96,%zmm25,%zmm24,%zmm10 // Fold MI into HI vextracti32x4 $1,%zmm10,%xmm4 vextracti32x4 $2,%zmm10,%xmm5 vextracti32x4 $3,%zmm10,%xmm6 @@ -677,32 +677,32 @@ vpternlogd $0x96,%xmm5,%xmm6,%xmm10 .Lcrypt_loop_4x_done__func1: - +// Check whether any data remains. testq %rdx,%rdx jz .Ldone__func1 +// The data length isn't a multiple of 256 bytes. Process the remaining +// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. +// Going one vector at a time may seem inefficient compared to having +// separate code paths for each possible number of vectors remaining. +// However, using a loop keeps the code size down, and it performs +// surprising well; modern CPUs will start executing the next iteration +// before the previous one finishes and also predict the number of loop +// iterations. For a similar reason, we roll up the AES rounds. +// +// On the last iteration, the remaining length may be less than 64 bytes. +// Handle this using masking. +// +// Since there are enough key powers available for all remaining data, +// there is no need to do a GHASH reduction after each iteration. +// Instead, multiply each remaining block by its own key power, and only +// do a GHASH reduction at the very end. - - - - - - - - - - - - - - - - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. movq %rdx,%rax negq %rax - andq $-16,%rax + andq $-16,%rax // -round_up(DATALEN, 16) leaq 256(%r9,%rax,1),%r8 vpxor %xmm4,%xmm4,%xmm4 vpxor %xmm5,%xmm5,%xmm5 @@ -712,9 +712,9 @@ jb .Lpartial_vec__func1 .Lcrypt_loop_1x__func1: +// Process a full 64-byte vector. - - +// Encrypt a vector of counter blocks. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpxord %zmm13,%zmm0,%zmm0 @@ -727,19 +727,19 @@ jne .Lvaesenc_loop_tail_full_vec__func1 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the vector of keystream blocks. vmovdqu8 (%rdi),%zmm1 vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi) - +// Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 (%r8),%zmm30 vpshufb %zmm8,%zmm0,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 @@ -757,10 +757,10 @@ jz .Lreduce__func1 .Lpartial_vec__func1: +// Process a partial vector of length 1 <= DATALEN < 64. - - - +// Set the data mask %k1 to DATALEN 1's. +// Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. movq $-1,%rax bzhiq %rdx,%rax,%rax kmovq %rax,%k1 @@ -770,8 +770,8 @@ bzhiq %rdx,%rax,%rax kmovq %rax,%k2 - - +// Encrypt one last vector of counter blocks. This does not need to be +// masked. The counter does not need to be incremented here. vpshufb %zmm8,%zmm12,%zmm0 vpxord %zmm13,%zmm0,%zmm0 leaq 16(%rcx),%rax @@ -783,38 +783,38 @@ jne .Lvaesenc_loop_tail_partialvec__func1 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the appropriate number of keystream bytes. vmovdqu8 (%rdi),%zmm1{%k1}{z} vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi){%k1} - - - - - - - - - - - - +// Update GHASH with the ciphertext block(s), without reducing. +// +// In the case of DATALEN < 64, the ciphertext is zero-padded to 64 +// bytes. (If decrypting, it's done by the above masked load. If +// encrypting, it's done by the below masked register-to-register move.) +// Note that if DATALEN <= 48, there will be additional padding beyond +// the padding of the last block specified by GHASH itself; i.e., there +// may be whole block(s) that get processed by the GHASH multiplication +// and reduction instructions but should not actually be included in the +// GHASH. However, any such blocks are all-zeroes, and the values that +// they're multiplied with are also all-zeroes. Therefore they just add +// 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 (%r8),%zmm30{%k2}{z} vmovdqu8 %zmm0,%zmm1{%k1}{z} vpshufb %zmm8,%zmm1,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 .Lreduce__func1: - +// Finally, do the GHASH reduction. vpclmulqdq $0x01,%zmm4,%zmm31,%zmm0 vpshufd $0x4e,%zmm4,%zmm4 vpternlogd $0x96,%zmm0,%zmm4,%zmm5 @@ -830,11 +830,11 @@ .Ldone__func1: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm8,%xmm10,%xmm10 vmovdqu %xmm10,(%r12) - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. popq %r12 .cfi_adjust_cfa_offset -8 .cfi_restore %r12 @@ -854,47 +854,47 @@ .cfi_adjust_cfa_offset 8 .cfi_offset %r12,-16 - movq 16(%rsp),%r12 - + movq 16(%rsp),%r12 // arg7 +// Load some constants. vbroadcasti32x4 .Lbswap_mask(%rip),%zmm8 vbroadcasti32x4 .Lgfpoly(%rip),%zmm31 - - +// Load the GHASH accumulator and the starting counter. +// BoringSSL passes these values in big endian format. vmovdqu (%r12),%xmm10 vpshufb %xmm8,%xmm10,%xmm10 vbroadcasti32x4 (%r8),%zmm12 vpshufb %zmm8,%zmm12,%zmm12 - - +// Load the AES key length in bytes. BoringSSL stores number of rounds +// minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. movl 240(%rcx),%r10d leal -20(,%r10,4),%r10d - - - +// Make RNDKEYLAST_PTR point to the last AES round key. This is the +// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +// respectively. Then load the zero-th and last round keys. leaq 96(%rcx,%r10,4),%r11 vbroadcasti32x4 (%rcx),%zmm13 vbroadcasti32x4 (%r11),%zmm14 - +// Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd .Lctr_pattern(%rip),%zmm12,%zmm12 - +// Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 .Linc_4blocks(%rip),%zmm11 - - +// If there are at least 256 bytes of data, then continue into the loop +// that processes 256 bytes of data at a time. Otherwise skip it. cmpq $256,%rdx jb .Lcrypt_loop_4x_done__func2 - +// Load powers of the hash key. vmovdqu8 256-256(%r9),%zmm27 vmovdqu8 256-192(%r9),%zmm28 vmovdqu8 256-128(%r9),%zmm29 vmovdqu8 256-64(%r9),%zmm30 - +// Cache as many additional AES round keys as possible. vbroadcasti32x4 -144(%r11),%zmm15 vbroadcasti32x4 -128(%r11),%zmm16 vbroadcasti32x4 -112(%r11),%zmm17 @@ -910,9 +910,9 @@ vmovdqu8 64(%rdi),%zmm5 vmovdqu8 128(%rdi),%zmm6 vmovdqu8 192(%rdi),%zmm7 - - - +// Start the AES encryption of the counter blocks. +// Increment le_ctr four times to generate four vectors of little-endian +// counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpshufb %zmm8,%zmm12,%zmm1 @@ -922,7 +922,7 @@ vpshufb %zmm8,%zmm12,%zmm3 vpaddd %zmm11,%zmm12,%zmm12 - +// AES "round zero": XOR in the zero-th round key. vpxord %zmm13,%zmm0,%zmm0 vpxord %zmm13,%zmm1,%zmm1 vpxord %zmm13,%zmm2,%zmm2 @@ -931,7 +931,7 @@ cmpl $24,%r10d jl .Laes128__func2 je .Laes192__func2 - +// AES-256 vbroadcasti32x4 -208(%r11),%zmm9 vaesenc %zmm9,%zmm0,%zmm0 vaesenc %zmm9,%zmm1,%zmm1 @@ -959,17 +959,17 @@ .Laes128__func2: - - - +// Prefetch the source data 512 bytes ahead into the L1 data cache, to +// improve performance when the hardware prefetcher is disabled. Assumes the +// L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 512+0(%rdi) prefetcht0 512+64(%rdi) prefetcht0 512+128(%rdi) prefetcht0 512+192(%rdi) - - - +// Finish the AES encryption of the counter blocks in AESDATA[0-3], +// interleaved with the GHASH update of the ciphertext blocks in +// GHASHDATA[0-3]. vpshufb %zmm8,%zmm4,%zmm4 vpxord %zmm10,%zmm4,%zmm4 vpshufb %zmm8,%zmm5,%zmm5 @@ -981,78 +981,78 @@ vaesenc %zmm15,%zmm3,%zmm3 vpshufb %zmm8,%zmm7,%zmm7 - vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 - vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 - vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 + vpclmulqdq $0x00,%zmm27,%zmm4,%zmm10 // LO_0 + vpclmulqdq $0x00,%zmm28,%zmm5,%zmm24 // LO_1 + vpclmulqdq $0x00,%zmm29,%zmm6,%zmm25 // LO_2 vaesenc %zmm16,%zmm0,%zmm0 vaesenc %zmm16,%zmm1,%zmm1 vaesenc %zmm16,%zmm2,%zmm2 vaesenc %zmm16,%zmm3,%zmm3 - vpxord %zmm24,%zmm10,%zmm10 - vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm10 - vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 + vpxord %zmm24,%zmm10,%zmm10 // sum(LO_{1,0}) + vpclmulqdq $0x00,%zmm30,%zmm7,%zmm26 // LO_3 + vpternlogd $0x96,%zmm26,%zmm25,%zmm10 // LO = sum(LO_{3,2,1,0}) + vpclmulqdq $0x01,%zmm27,%zmm4,%zmm24 // MI_0 vaesenc %zmm17,%zmm0,%zmm0 vaesenc %zmm17,%zmm1,%zmm1 vaesenc %zmm17,%zmm2,%zmm2 vaesenc %zmm17,%zmm3,%zmm3 - vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 + vpclmulqdq $0x01,%zmm28,%zmm5,%zmm25 // MI_1 + vpclmulqdq $0x01,%zmm29,%zmm6,%zmm26 // MI_2 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{2,1,0}) + vpclmulqdq $0x01,%zmm30,%zmm7,%zmm25 // MI_3 vaesenc %zmm18,%zmm0,%zmm0 vaesenc %zmm18,%zmm1,%zmm1 vaesenc %zmm18,%zmm2,%zmm2 vaesenc %zmm18,%zmm3,%zmm3 - vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 - vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 + vpclmulqdq $0x10,%zmm27,%zmm4,%zmm26 // MI_4 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{4,3,2,1,0}) + vpclmulqdq $0x10,%zmm28,%zmm5,%zmm25 // MI_5 + vpclmulqdq $0x10,%zmm29,%zmm6,%zmm26 // MI_6 vaesenc %zmm19,%zmm0,%zmm0 vaesenc %zmm19,%zmm1,%zmm1 vaesenc %zmm19,%zmm2,%zmm2 vaesenc %zmm19,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm25,%zmm24 - vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 - vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 - vpxord %zmm25,%zmm24,%zmm24 + vpternlogd $0x96,%zmm26,%zmm25,%zmm24 // sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq $0x01,%zmm10,%zmm31,%zmm26 // LO_L*(x^63 + x^62 + x^57) + vpclmulqdq $0x10,%zmm30,%zmm7,%zmm25 // MI_7 + vpxord %zmm25,%zmm24,%zmm24 // MI = sum(MI_{7,6,5,4,3,2,1,0}) vaesenc %zmm20,%zmm0,%zmm0 vaesenc %zmm20,%zmm1,%zmm1 vaesenc %zmm20,%zmm2,%zmm2 vaesenc %zmm20,%zmm3,%zmm3 - vpshufd $0x4e,%zmm10,%zmm10 - vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 - vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 - vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 + vpshufd $0x4e,%zmm10,%zmm10 // Swap halves of LO + vpclmulqdq $0x11,%zmm27,%zmm4,%zmm4 // HI_0 + vpclmulqdq $0x11,%zmm28,%zmm5,%zmm5 // HI_1 + vpclmulqdq $0x11,%zmm29,%zmm6,%zmm6 // HI_2 vaesenc %zmm21,%zmm0,%zmm0 vaesenc %zmm21,%zmm1,%zmm1 vaesenc %zmm21,%zmm2,%zmm2 vaesenc %zmm21,%zmm3,%zmm3 - vpternlogd $0x96,%zmm26,%zmm10,%zmm24 - vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 - vpternlogd $0x96,%zmm6,%zmm5,%zmm4 - vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 + vpternlogd $0x96,%zmm26,%zmm10,%zmm24 // Fold LO into MI + vpclmulqdq $0x11,%zmm30,%zmm7,%zmm7 // HI_3 + vpternlogd $0x96,%zmm6,%zmm5,%zmm4 // sum(HI_{2,1,0}) + vpclmulqdq $0x01,%zmm24,%zmm31,%zmm25 // MI_L*(x^63 + x^62 + x^57) vaesenc %zmm22,%zmm0,%zmm0 vaesenc %zmm22,%zmm1,%zmm1 vaesenc %zmm22,%zmm2,%zmm2 vaesenc %zmm22,%zmm3,%zmm3 - vpxord %zmm7,%zmm4,%zmm10 - vpshufd $0x4e,%zmm24,%zmm24 - vpternlogd $0x96,%zmm25,%zmm24,%zmm10 + vpxord %zmm7,%zmm4,%zmm10 // HI = sum(HI_{3,2,1,0}) + vpshufd $0x4e,%zmm24,%zmm24 // Swap halves of MI + vpternlogd $0x96,%zmm25,%zmm24,%zmm10 // Fold MI into HI vaesenc %zmm23,%zmm0,%zmm0 vaesenc %zmm23,%zmm1,%zmm1 @@ -1085,32 +1085,32 @@ cmpq $256,%rdx jae .Lcrypt_loop_4x__func2 .Lcrypt_loop_4x_done__func2: - +// Check whether any data remains. testq %rdx,%rdx jz .Ldone__func2 +// The data length isn't a multiple of 256 bytes. Process the remaining +// data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. +// Going one vector at a time may seem inefficient compared to having +// separate code paths for each possible number of vectors remaining. +// However, using a loop keeps the code size down, and it performs +// surprising well; modern CPUs will start executing the next iteration +// before the previous one finishes and also predict the number of loop +// iterations. For a similar reason, we roll up the AES rounds. +// +// On the last iteration, the remaining length may be less than 64 bytes. +// Handle this using masking. +// +// Since there are enough key powers available for all remaining data, +// there is no need to do a GHASH reduction after each iteration. +// Instead, multiply each remaining block by its own key power, and only +// do a GHASH reduction at the very end. - - - - - - - - - - - - - - - - - - +// Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +// is the number of blocks that remain. movq %rdx,%rax negq %rax - andq $-16,%rax + andq $-16,%rax // -round_up(DATALEN, 16) leaq 256(%r9,%rax,1),%r8 vpxor %xmm4,%xmm4,%xmm4 vpxor %xmm5,%xmm5,%xmm5 @@ -1120,9 +1120,9 @@ jb .Lpartial_vec__func2 .Lcrypt_loop_1x__func2: +// Process a full 64-byte vector. - - +// Encrypt a vector of counter blocks. vpshufb %zmm8,%zmm12,%zmm0 vpaddd %zmm11,%zmm12,%zmm12 vpxord %zmm13,%zmm0,%zmm0 @@ -1135,19 +1135,19 @@ jne .Lvaesenc_loop_tail_full_vec__func2 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the vector of keystream blocks. vmovdqu8 (%rdi),%zmm1 vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi) - +// Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 (%r8),%zmm30 vpshufb %zmm8,%zmm1,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 @@ -1165,10 +1165,10 @@ jz .Lreduce__func2 .Lpartial_vec__func2: +// Process a partial vector of length 1 <= DATALEN < 64. - - - +// Set the data mask %k1 to DATALEN 1's. +// Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. movq $-1,%rax bzhiq %rdx,%rax,%rax kmovq %rax,%k1 @@ -1178,8 +1178,8 @@ bzhiq %rdx,%rax,%rax kmovq %rax,%k2 - - +// Encrypt one last vector of counter blocks. This does not need to be +// masked. The counter does not need to be incremented here. vpshufb %zmm8,%zmm12,%zmm0 vpxord %zmm13,%zmm0,%zmm0 leaq 16(%rcx),%rax @@ -1191,38 +1191,38 @@ jne .Lvaesenc_loop_tail_partialvec__func2 vaesenclast %zmm14,%zmm0,%zmm0 - +// XOR the data with the appropriate number of keystream bytes. vmovdqu8 (%rdi),%zmm1{%k1}{z} vpxord %zmm1,%zmm0,%zmm0 vmovdqu8 %zmm0,(%rsi){%k1} - - - - - - - - - - - - +// Update GHASH with the ciphertext block(s), without reducing. +// +// In the case of DATALEN < 64, the ciphertext is zero-padded to 64 +// bytes. (If decrypting, it's done by the above masked load. If +// encrypting, it's done by the below masked register-to-register move.) +// Note that if DATALEN <= 48, there will be additional padding beyond +// the padding of the last block specified by GHASH itself; i.e., there +// may be whole block(s) that get processed by the GHASH multiplication +// and reduction instructions but should not actually be included in the +// GHASH. However, any such blocks are all-zeroes, and the values that +// they're multiplied with are also all-zeroes. Therefore they just add +// 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 (%r8),%zmm30{%k2}{z} vpshufb %zmm8,%zmm1,%zmm0 vpxord %zmm10,%zmm0,%zmm0 - vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 - vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 - vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 - vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 + vpclmulqdq $0x00,%zmm30,%zmm0,%zmm7 // a_L * b_L + vpclmulqdq $0x01,%zmm30,%zmm0,%zmm1 // a_L * b_H + vpclmulqdq $0x10,%zmm30,%zmm0,%zmm2 // a_H * b_L + vpclmulqdq $0x11,%zmm30,%zmm0,%zmm3 // a_H * b_H vpxord %zmm7,%zmm4,%zmm4 vpternlogd $0x96,%zmm2,%zmm1,%zmm5 vpxord %zmm3,%zmm6,%zmm6 .Lreduce__func2: - +// Finally, do the GHASH reduction. vpclmulqdq $0x01,%zmm4,%zmm31,%zmm0 vpshufd $0x4e,%zmm4,%zmm4 vpternlogd $0x96,%zmm0,%zmm4,%zmm5 @@ -1238,11 +1238,11 @@ .Ldone__func2: - +// Store the updated GHASH accumulator back to memory. vpshufb %xmm8,%xmm10,%xmm10 vmovdqu %xmm10,(%r12) - vzeroupper + vzeroupper // This is needed after using ymm or zmm registers. popq %r12 .cfi_adjust_cfa_offset -8 .cfi_restore %r12
diff --git a/gen/bcm/aes-gcm-avx512-x86_64-win.asm b/gen/bcm/aes-gcm-avx512-x86_64-win.asm index 7673791..17e44e6 100644 --- a/gen/bcm/aes-gcm-avx512-x86_64-win.asm +++ b/gen/bcm/aes-gcm-avx512-x86_64-win.asm
@@ -16,32 +16,32 @@ ALIGN 64 aes_gcm_avx512_constants: - +; A shuffle mask that reflects the bytes of 16-byte blocks $L$bswap_mask: DQ 0x08090a0b0c0d0e0f,0x0001020304050607 - - - - - - - +; This is the GHASH reducing polynomial without its constant term, i.e. +; x^128 + x^7 + x^2 + x, represented using the backwards mapping +; between bits and polynomial coefficients. +; +; Alternatively, it can be interpreted as the naturally-ordered +; representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the +; "reversed" GHASH reducing polynomial without its x^128 term. $L$gfpoly: DQ 1,0xc200000000000000 - +; Same as above, but with the (1 << 64) bit set. $L$gfpoly_and_internal_carrybit: DQ 1,0xc200000000000001 - +; Values needed to prepare the initial vector of counter blocks. $L$ctr_pattern: DQ 0,0 DQ 1,0 DQ 2,0 DQ 3,0 - +; The number of AES blocks per vector, as a 128-bit value. $L$inc_4blocks: DQ 4,0 @@ -54,100 +54,100 @@ _CET_ENDBR - +; Get pointer to lowest set of key powers (located at end of array). lea r8,[((256-64))+rcx] - - +; Load the byte-reflected hash subkey. BoringSSL provides it in +; byte-reflected form except the two halves are in the wrong order. vpshufd xmm3,XMMWORD[rdx],0x4e - - - - - - - - - - - - - - - +; Finish preprocessing the first key power, H^1. Since this GHASH +; implementation operates directly on values with the backwards bit +; order specified by the GCM standard, it's necessary to preprocess the +; raw key as follows. First, reflect its bytes. Second, multiply it +; by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards +; interpretation of polynomial coefficients), which can also be +; interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121 +; + 1 using the alternative, natural interpretation of polynomial +; coefficients. For details, see the comment above _ghash_mul. +; +; Either way, for the multiplication the concrete operation performed +; is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2 +; << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit +; wide shift instruction, so instead double each of the two 64-bit +; halves and incorporate the internal carry bit into the value XOR'd. vpshufd xmm0,xmm3,0xd3 vpsrad xmm0,xmm0,31 vpaddq xmm3,xmm3,xmm3 - +; H_CUR_XMM ^= TMP0_XMM & gfpoly_and_internal_carrybit vpternlogd xmm3,xmm0,XMMWORD[$L$gfpoly_and_internal_carrybit],0x78 - +; Load the gfpoly constant. vbroadcasti32x4 zmm5,ZMMWORD[$L$gfpoly] +; Square H^1 to get H^2. +; +; Note that as with H^1, all higher key powers also need an extra +; factor of x^-1 (or x using the natural interpretation). Nothing +; special needs to be done to make this happen, though: H^1 * H^1 would +; end up with two factors of x^-1, but the multiplication consumes one. +; So the product H^2 ends up with the desired one factor of x^-1. + vpclmulqdq xmm0,xmm3,xmm3,0x00 ; LO = a_L * a_L + vpclmulqdq xmm4,xmm3,xmm3,0x11 ; HI = a_H * a_H + vpclmulqdq xmm1,xmm5,xmm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd xmm0,xmm0,0x4e ; Swap halves of LO + vpxor xmm1,xmm1,xmm0 ; Fold LO into MI + vpclmulqdq xmm0,xmm5,xmm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd xmm1,xmm1,0x4e ; Swap halves of MI + vpternlogd xmm4,xmm1,xmm0,0x96 ; Fold MI into HI - - - - - - vpclmulqdq xmm0,xmm3,xmm3,0x00 - vpclmulqdq xmm4,xmm3,xmm3,0x11 - vpclmulqdq xmm1,xmm5,xmm0,0x01 - vpshufd xmm0,xmm0,0x4e - vpxor xmm1,xmm1,xmm0 - vpclmulqdq xmm0,xmm5,xmm1,0x01 - vpshufd xmm1,xmm1,0x4e - vpternlogd xmm4,xmm1,xmm0,0x96 - - - +; Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2]. vinserti128 ymm3,ymm4,xmm3,1 vinserti128 ymm4,ymm4,xmm4,1 - - vpclmulqdq ymm0,ymm3,ymm4,0x00 - vpclmulqdq ymm1,ymm3,ymm4,0x01 - vpclmulqdq ymm2,ymm3,ymm4,0x10 - vpxord ymm1,ymm1,ymm2 - vpclmulqdq ymm2,ymm5,ymm0,0x01 - vpshufd ymm0,ymm0,0x4e - vpternlogd ymm1,ymm0,ymm2,0x96 - vpclmulqdq ymm4,ymm3,ymm4,0x11 - vpclmulqdq ymm0,ymm5,ymm1,0x01 - vpshufd ymm1,ymm1,0x4e - vpternlogd ymm4,ymm1,ymm0,0x96 +; Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4]. + vpclmulqdq ymm0,ymm3,ymm4,0x00 ; LO = a_L * b_L + vpclmulqdq ymm1,ymm3,ymm4,0x01 ; MI_0 = a_L * b_H + vpclmulqdq ymm2,ymm3,ymm4,0x10 ; MI_1 = a_H * b_L + vpxord ymm1,ymm1,ymm2 ; MI = MI_0 + MI_1 + vpclmulqdq ymm2,ymm5,ymm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd ymm0,ymm0,0x4e ; Swap halves of LO + vpternlogd ymm1,ymm0,ymm2,0x96 ; Fold LO into MI + vpclmulqdq ymm4,ymm3,ymm4,0x11 ; HI = a_H * b_H + vpclmulqdq ymm0,ymm5,ymm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd ymm1,ymm1,0x4e ; Swap halves of MI + vpternlogd ymm4,ymm1,ymm0,0x96 ; Fold MI into HI vinserti64x4 zmm3,zmm4,ymm3,1 vshufi64x2 zmm4,zmm4,zmm4,0 - +; Store the lowest set of key powers. vmovdqu8 ZMMWORD[r8],zmm3 - - - +; Compute and store the remaining key powers. +; Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by +; [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)]. mov eax,3 $L$precompute_next: sub r8,64 - vpclmulqdq zmm0,zmm3,zmm4,0x00 - vpclmulqdq zmm1,zmm3,zmm4,0x01 - vpclmulqdq zmm2,zmm3,zmm4,0x10 - vpxord zmm1,zmm1,zmm2 - vpclmulqdq zmm2,zmm5,zmm0,0x01 - vpshufd zmm0,zmm0,0x4e - vpternlogd zmm1,zmm0,zmm2,0x96 - vpclmulqdq zmm3,zmm3,zmm4,0x11 - vpclmulqdq zmm0,zmm5,zmm1,0x01 - vpshufd zmm1,zmm1,0x4e - vpternlogd zmm3,zmm1,zmm0,0x96 + vpclmulqdq zmm0,zmm3,zmm4,0x00 ; LO = a_L * b_L + vpclmulqdq zmm1,zmm3,zmm4,0x01 ; MI_0 = a_L * b_H + vpclmulqdq zmm2,zmm3,zmm4,0x10 ; MI_1 = a_H * b_L + vpxord zmm1,zmm1,zmm2 ; MI = MI_0 + MI_1 + vpclmulqdq zmm2,zmm5,zmm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd zmm0,zmm0,0x4e ; Swap halves of LO + vpternlogd zmm1,zmm0,zmm2,0x96 ; Fold LO into MI + vpclmulqdq zmm3,zmm3,zmm4,0x11 ; HI = a_H * b_H + vpclmulqdq zmm0,zmm5,zmm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd zmm1,zmm1,0x4e ; Swap halves of MI + vpternlogd zmm3,zmm1,zmm0,0x96 ; Fold MI into HI vmovdqu8 ZMMWORD[r8],zmm3 dec eax jnz NEAR $L$precompute_next - vzeroupper + vzeroupper ; This is needed after using ymm or zmm registers. ret @@ -172,23 +172,23 @@ vmovdqu xmm3,XMMWORD[$L$gfpoly] vpshufb xmm0,xmm0,xmm1 - vpclmulqdq xmm4,xmm0,xmm2,0x00 - vpclmulqdq xmm5,xmm0,xmm2,0x01 - vpclmulqdq xmm6,xmm0,xmm2,0x10 - vpxord xmm5,xmm5,xmm6 - vpclmulqdq xmm6,xmm3,xmm4,0x01 - vpshufd xmm4,xmm4,0x4e - vpternlogd xmm5,xmm4,xmm6,0x96 - vpclmulqdq xmm0,xmm0,xmm2,0x11 - vpclmulqdq xmm4,xmm3,xmm5,0x01 - vpshufd xmm5,xmm5,0x4e - vpternlogd xmm0,xmm5,xmm4,0x96 + vpclmulqdq xmm4,xmm0,xmm2,0x00 ; LO = a_L * b_L + vpclmulqdq xmm5,xmm0,xmm2,0x01 ; MI_0 = a_L * b_H + vpclmulqdq xmm6,xmm0,xmm2,0x10 ; MI_1 = a_H * b_L + vpxord xmm5,xmm5,xmm6 ; MI = MI_0 + MI_1 + vpclmulqdq xmm6,xmm3,xmm4,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd xmm4,xmm4,0x4e ; Swap halves of LO + vpternlogd xmm5,xmm4,xmm6,0x96 ; Fold LO into MI + vpclmulqdq xmm0,xmm0,xmm2,0x11 ; HI = a_H * b_H + vpclmulqdq xmm4,xmm3,xmm5,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd xmm5,xmm5,0x4e ; Swap halves of MI + vpternlogd xmm0,xmm5,xmm4,0x96 ; Fold MI into HI vpshufb xmm0,xmm0,xmm1 vmovdqu XMMWORD[rcx],xmm0 - +; No need for vzeroupper, since only xmm registers were used. vmovdqa xmm6,XMMWORD[rsp] add rsp,24 ret @@ -223,37 +223,37 @@ $L$SEH_endprologue_gcm_ghash_vpclmulqdq_avx512_11: - - - +; Load the bswap_mask and gfpoly constants. Since AADLEN is usually small, +; usually only 128-bit vectors will be used. So as an optimization, don't +; broadcast these constants to all 128-bit lanes quite yet. vmovdqu xmm4,XMMWORD[$L$bswap_mask] vmovdqu xmm10,XMMWORD[$L$gfpoly] - +; Load the GHASH accumulator. vmovdqu xmm5,XMMWORD[rcx] vpshufb xmm5,xmm5,xmm4 - +; Optimize for AADLEN < 64 by checking for AADLEN < 64 before AADLEN < 256. cmp r9,64 jb NEAR $L$aad_blockbyblock - - +; AADLEN >= 64, so we'll operate on full vectors. Broadcast bswap_mask and +; gfpoly to all 128-bit lanes. vshufi64x2 zmm4,zmm4,zmm4,0 vshufi64x2 zmm10,zmm10,zmm10,0 - +; Load the lowest set of key powers. vmovdqu8 zmm9,ZMMWORD[((256-64))+rdx] cmp r9,256 jb NEAR $L$aad_loop_1x - +; AADLEN >= 256. Load the higher key powers. vmovdqu8 zmm6,ZMMWORD[((256-256))+rdx] vmovdqu8 zmm7,ZMMWORD[((256-192))+rdx] vmovdqu8 zmm8,ZMMWORD[((256-128))+rdx] - +; Update GHASH with 256 bytes of AAD at a time. $L$aad_loop_4x: vmovdqu8 zmm0,ZMMWORD[r8] vmovdqu8 zmm1,ZMMWORD[64+r8] @@ -264,36 +264,36 @@ vpshufb zmm1,zmm1,zmm4 vpshufb zmm2,zmm2,zmm4 vpshufb zmm3,zmm3,zmm4 - vpclmulqdq zmm5,zmm0,zmm6,0x00 - vpclmulqdq zmm11,zmm1,zmm7,0x00 - vpclmulqdq zmm12,zmm2,zmm8,0x00 - vpxord zmm5,zmm5,zmm11 - vpclmulqdq zmm13,zmm3,zmm9,0x00 - vpternlogd zmm5,zmm12,zmm13,0x96 - vpclmulqdq zmm11,zmm0,zmm6,0x01 - vpclmulqdq zmm12,zmm1,zmm7,0x01 - vpclmulqdq zmm13,zmm2,zmm8,0x01 - vpternlogd zmm11,zmm12,zmm13,0x96 - vpclmulqdq zmm12,zmm3,zmm9,0x01 - vpclmulqdq zmm13,zmm0,zmm6,0x10 - vpternlogd zmm11,zmm12,zmm13,0x96 - vpclmulqdq zmm12,zmm1,zmm7,0x10 - vpclmulqdq zmm13,zmm2,zmm8,0x10 - vpternlogd zmm11,zmm12,zmm13,0x96 - vpclmulqdq zmm13,zmm10,zmm5,0x01 - vpclmulqdq zmm12,zmm3,zmm9,0x10 - vpxord zmm11,zmm11,zmm12 - vpshufd zmm5,zmm5,0x4e - vpclmulqdq zmm0,zmm0,zmm6,0x11 - vpclmulqdq zmm1,zmm1,zmm7,0x11 - vpclmulqdq zmm2,zmm2,zmm8,0x11 - vpternlogd zmm11,zmm5,zmm13,0x96 - vpclmulqdq zmm3,zmm3,zmm9,0x11 - vpternlogd zmm0,zmm1,zmm2,0x96 - vpclmulqdq zmm12,zmm10,zmm11,0x01 - vpxord zmm5,zmm0,zmm3 - vpshufd zmm11,zmm11,0x4e - vpternlogd zmm5,zmm11,zmm12,0x96 + vpclmulqdq zmm5,zmm0,zmm6,0x00 ; LO_0 + vpclmulqdq zmm11,zmm1,zmm7,0x00 ; LO_1 + vpclmulqdq zmm12,zmm2,zmm8,0x00 ; LO_2 + vpxord zmm5,zmm5,zmm11 ; sum(LO_{1,0}) + vpclmulqdq zmm13,zmm3,zmm9,0x00 ; LO_3 + vpternlogd zmm5,zmm12,zmm13,0x96 ; LO = sum(LO_{3,2,1,0}) + vpclmulqdq zmm11,zmm0,zmm6,0x01 ; MI_0 + vpclmulqdq zmm12,zmm1,zmm7,0x01 ; MI_1 + vpclmulqdq zmm13,zmm2,zmm8,0x01 ; MI_2 + vpternlogd zmm11,zmm12,zmm13,0x96 ; sum(MI_{2,1,0}) + vpclmulqdq zmm12,zmm3,zmm9,0x01 ; MI_3 + vpclmulqdq zmm13,zmm0,zmm6,0x10 ; MI_4 + vpternlogd zmm11,zmm12,zmm13,0x96 ; sum(MI_{4,3,2,1,0}) + vpclmulqdq zmm12,zmm1,zmm7,0x10 ; MI_5 + vpclmulqdq zmm13,zmm2,zmm8,0x10 ; MI_6 + vpternlogd zmm11,zmm12,zmm13,0x96 ; sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq zmm13,zmm10,zmm5,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpclmulqdq zmm12,zmm3,zmm9,0x10 ; MI_7 + vpxord zmm11,zmm11,zmm12 ; MI = sum(MI_{7,6,5,4,3,2,1,0}) + vpshufd zmm5,zmm5,0x4e ; Swap halves of LO + vpclmulqdq zmm0,zmm0,zmm6,0x11 ; HI_0 + vpclmulqdq zmm1,zmm1,zmm7,0x11 ; HI_1 + vpclmulqdq zmm2,zmm2,zmm8,0x11 ; HI_2 + vpternlogd zmm11,zmm5,zmm13,0x96 ; Fold LO into MI + vpclmulqdq zmm3,zmm3,zmm9,0x11 ; HI_3 + vpternlogd zmm0,zmm1,zmm2,0x96 ; sum(HI_{2,1,0}) + vpclmulqdq zmm12,zmm10,zmm11,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpxord zmm5,zmm0,zmm3 ; HI = sum(HI_{3,2,1,0}) + vpshufd zmm11,zmm11,0x4e ; Swap halves of MI + vpternlogd zmm5,zmm11,zmm12,0x96 ; Fold MI into HI vextracti32x4 xmm0,zmm5,1 vextracti32x4 xmm1,zmm5,2 vextracti32x4 xmm2,zmm5,3 @@ -305,24 +305,24 @@ cmp r9,256 jae NEAR $L$aad_loop_4x - +; Update GHASH with 64 bytes of AAD at a time. cmp r9,64 jb NEAR $L$aad_large_done $L$aad_loop_1x: vmovdqu8 zmm0,ZMMWORD[r8] vpshufb zmm0,zmm0,zmm4 vpxord zmm5,zmm5,zmm0 - vpclmulqdq zmm0,zmm5,zmm9,0x00 - vpclmulqdq zmm1,zmm5,zmm9,0x01 - vpclmulqdq zmm2,zmm5,zmm9,0x10 - vpxord zmm1,zmm1,zmm2 - vpclmulqdq zmm2,zmm10,zmm0,0x01 - vpshufd zmm0,zmm0,0x4e - vpternlogd zmm1,zmm0,zmm2,0x96 - vpclmulqdq zmm5,zmm5,zmm9,0x11 - vpclmulqdq zmm0,zmm10,zmm1,0x01 - vpshufd zmm1,zmm1,0x4e - vpternlogd zmm5,zmm1,zmm0,0x96 + vpclmulqdq zmm0,zmm5,zmm9,0x00 ; LO = a_L * b_L + vpclmulqdq zmm1,zmm5,zmm9,0x01 ; MI_0 = a_L * b_H + vpclmulqdq zmm2,zmm5,zmm9,0x10 ; MI_1 = a_H * b_L + vpxord zmm1,zmm1,zmm2 ; MI = MI_0 + MI_1 + vpclmulqdq zmm2,zmm10,zmm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd zmm0,zmm0,0x4e ; Swap halves of LO + vpternlogd zmm1,zmm0,zmm2,0x96 ; Fold LO into MI + vpclmulqdq zmm5,zmm5,zmm9,0x11 ; HI = a_H * b_H + vpclmulqdq zmm0,zmm10,zmm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd zmm1,zmm1,0x4e ; Swap halves of MI + vpternlogd zmm5,zmm1,zmm0,0x96 ; Fold MI into HI vextracti32x4 xmm0,zmm5,1 vextracti32x4 xmm1,zmm5,2 @@ -337,7 +337,7 @@ $L$aad_large_done: - +; GHASH the remaining data 16 bytes at a time, using xmm registers only. $L$aad_blockbyblock: test r9,r9 jz NEAR $L$aad_done @@ -346,28 +346,28 @@ vmovdqu xmm0,XMMWORD[r8] vpshufb xmm0,xmm0,xmm4 vpxor xmm5,xmm5,xmm0 - vpclmulqdq xmm0,xmm5,xmm9,0x00 - vpclmulqdq xmm1,xmm5,xmm9,0x01 - vpclmulqdq xmm2,xmm5,xmm9,0x10 - vpxord xmm1,xmm1,xmm2 - vpclmulqdq xmm2,xmm10,xmm0,0x01 - vpshufd xmm0,xmm0,0x4e - vpternlogd xmm1,xmm0,xmm2,0x96 - vpclmulqdq xmm5,xmm5,xmm9,0x11 - vpclmulqdq xmm0,xmm10,xmm1,0x01 - vpshufd xmm1,xmm1,0x4e - vpternlogd xmm5,xmm1,xmm0,0x96 + vpclmulqdq xmm0,xmm5,xmm9,0x00 ; LO = a_L * b_L + vpclmulqdq xmm1,xmm5,xmm9,0x01 ; MI_0 = a_L * b_H + vpclmulqdq xmm2,xmm5,xmm9,0x10 ; MI_1 = a_H * b_L + vpxord xmm1,xmm1,xmm2 ; MI = MI_0 + MI_1 + vpclmulqdq xmm2,xmm10,xmm0,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpshufd xmm0,xmm0,0x4e ; Swap halves of LO + vpternlogd xmm1,xmm0,xmm2,0x96 ; Fold LO into MI + vpclmulqdq xmm5,xmm5,xmm9,0x11 ; HI = a_H * b_H + vpclmulqdq xmm0,xmm10,xmm1,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpshufd xmm1,xmm1,0x4e ; Swap halves of MI + vpternlogd xmm5,xmm1,xmm0,0x96 ; Fold MI into HI add r8,16 sub r9,16 jnz NEAR $L$aad_loop_blockbyblock $L$aad_done: - +; Store the updated GHASH accumulator back to memory. vpshufb xmm5,xmm5,xmm4 vmovdqu XMMWORD[rcx],xmm5 - vzeroupper + vzeroupper ; This is needed after using ymm or zmm registers. vmovdqa xmm6,XMMWORD[rsp] vmovdqa xmm7,XMMWORD[16+rsp] vmovdqa xmm8,XMMWORD[32+rsp] @@ -395,9 +395,9 @@ push r12 $L$SEH_prologue_aes_gcm_enc_update_vaes_avx512_4: - mov rsi,QWORD[64+rsp] - mov rdi,QWORD[72+rsp] - mov r12,QWORD[80+rsp] + mov rsi,QWORD[64+rsp] ; arg5 + mov rdi,QWORD[72+rsp] ; arg6 + mov r12,QWORD[80+rsp] ; arg7 sub rsp,160 $L$SEH_prologue_aes_gcm_enc_update_vaes_avx512_5: vmovdqa XMMWORD[rsp],xmm6 @@ -426,49 +426,49 @@ EXTERN BORINGSSL_function_hit mov BYTE[((BORINGSSL_function_hit+7))],1 %endif - +; Load some constants. vbroadcasti32x4 zmm8,ZMMWORD[$L$bswap_mask] vbroadcasti32x4 zmm31,ZMMWORD[$L$gfpoly] - - +; Load the GHASH accumulator and the starting counter. +; BoringSSL passes these values in big endian format. vmovdqu xmm10,XMMWORD[r12] vpshufb xmm10,xmm10,xmm8 vbroadcasti32x4 zmm12,ZMMWORD[rsi] vpshufb zmm12,zmm12,zmm8 - - +; Load the AES key length in bytes. BoringSSL stores number of rounds +; minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. mov r10d,DWORD[240+r9] lea r10d,[((-20))+r10*4] - - - +; Make RNDKEYLAST_PTR point to the last AES round key. This is the +; round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +; respectively. Then load the zero-th and last round keys. lea r11,[96+r10*4+r9] vbroadcasti32x4 zmm13,ZMMWORD[r9] vbroadcasti32x4 zmm14,ZMMWORD[r11] - +; Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd zmm12,zmm12,ZMMWORD[$L$ctr_pattern] - +; Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 zmm11,ZMMWORD[$L$inc_4blocks] - - +; If there are at least 256 bytes of data, then continue into the loop +; that processes 256 bytes of data at a time. Otherwise skip it. cmp r8,256 jb NEAR $L$crypt_loop_4x_done__func1 - +; Load powers of the hash key. vmovdqu8 zmm27,ZMMWORD[((256-256))+rdi] vmovdqu8 zmm28,ZMMWORD[((256-192))+rdi] vmovdqu8 zmm29,ZMMWORD[((256-128))+rdi] vmovdqu8 zmm30,ZMMWORD[((256-64))+rdi] - - - - +; Encrypt the first 4 vectors of plaintext blocks. Leave the resulting +; ciphertext in GHASHDATA[0-3] for GHASH. +; Increment le_ctr four times to generate four vectors of little-endian +; counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb zmm0,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 vpshufb zmm1,zmm12,zmm8 @@ -478,7 +478,7 @@ vpshufb zmm3,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 - +; AES "round zero": XOR in the zero-th round key. vpxord zmm0,zmm0,zmm13 vpxord zmm1,zmm1,zmm13 vpxord zmm2,zmm2,zmm13 @@ -513,7 +513,7 @@ sub r8,256 cmp r8,256 jb NEAR $L$ghash_last_ciphertext_4x__func1 - +; Cache as many additional AES round keys as possible. vbroadcasti32x4 zmm15,ZMMWORD[((-144))+r11] vbroadcasti32x4 zmm16,ZMMWORD[((-128))+r11] vbroadcasti32x4 zmm17,ZMMWORD[((-112))+r11] @@ -525,9 +525,9 @@ vbroadcasti32x4 zmm23,ZMMWORD[((-16))+r11] $L$crypt_loop_4x__func1: - - - +; Start the AES encryption of the counter blocks. +; Increment le_ctr four times to generate four vectors of little-endian +; counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb zmm0,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 vpshufb zmm1,zmm12,zmm8 @@ -537,7 +537,7 @@ vpshufb zmm3,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 - +; AES "round zero": XOR in the zero-th round key. vpxord zmm0,zmm0,zmm13 vpxord zmm1,zmm1,zmm13 vpxord zmm2,zmm2,zmm13 @@ -546,7 +546,7 @@ cmp r10d,24 jl NEAR $L$aes128__func1 je NEAR $L$aes192__func1 - +; AES-256 vbroadcasti32x4 zmm9,ZMMWORD[((-208))+r11] vaesenc zmm0,zmm0,zmm9 vaesenc zmm1,zmm1,zmm9 @@ -574,17 +574,17 @@ $L$aes128__func1: - - - +; Prefetch the source data 512 bytes ahead into the L1 data cache, to +; improve performance when the hardware prefetcher is disabled. Assumes the +; L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 [((512+0))+rcx] prefetcht0 [((512+64))+rcx] prefetcht0 [((512+128))+rcx] prefetcht0 [((512+192))+rcx] - - - +; Finish the AES encryption of the counter blocks in AESDATA[0-3], +; interleaved with the GHASH update of the ciphertext blocks in +; GHASHDATA[0-3]. vpshufb zmm4,zmm4,zmm8 vpxord zmm4,zmm4,zmm10 vpshufb zmm5,zmm5,zmm8 @@ -596,78 +596,78 @@ vaesenc zmm3,zmm3,zmm15 vpshufb zmm7,zmm7,zmm8 - vpclmulqdq zmm10,zmm4,zmm27,0x00 - vpclmulqdq zmm24,zmm5,zmm28,0x00 - vpclmulqdq zmm25,zmm6,zmm29,0x00 + vpclmulqdq zmm10,zmm4,zmm27,0x00 ; LO_0 + vpclmulqdq zmm24,zmm5,zmm28,0x00 ; LO_1 + vpclmulqdq zmm25,zmm6,zmm29,0x00 ; LO_2 vaesenc zmm0,zmm0,zmm16 vaesenc zmm1,zmm1,zmm16 vaesenc zmm2,zmm2,zmm16 vaesenc zmm3,zmm3,zmm16 - vpxord zmm10,zmm10,zmm24 - vpclmulqdq zmm26,zmm7,zmm30,0x00 - vpternlogd zmm10,zmm25,zmm26,0x96 - vpclmulqdq zmm24,zmm4,zmm27,0x01 + vpxord zmm10,zmm10,zmm24 ; sum(LO_{1,0}) + vpclmulqdq zmm26,zmm7,zmm30,0x00 ; LO_3 + vpternlogd zmm10,zmm25,zmm26,0x96 ; LO = sum(LO_{3,2,1,0}) + vpclmulqdq zmm24,zmm4,zmm27,0x01 ; MI_0 vaesenc zmm0,zmm0,zmm17 vaesenc zmm1,zmm1,zmm17 vaesenc zmm2,zmm2,zmm17 vaesenc zmm3,zmm3,zmm17 - vpclmulqdq zmm25,zmm5,zmm28,0x01 - vpclmulqdq zmm26,zmm6,zmm29,0x01 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm25,zmm7,zmm30,0x01 + vpclmulqdq zmm25,zmm5,zmm28,0x01 ; MI_1 + vpclmulqdq zmm26,zmm6,zmm29,0x01 ; MI_2 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{2,1,0}) + vpclmulqdq zmm25,zmm7,zmm30,0x01 ; MI_3 vaesenc zmm0,zmm0,zmm18 vaesenc zmm1,zmm1,zmm18 vaesenc zmm2,zmm2,zmm18 vaesenc zmm3,zmm3,zmm18 - vpclmulqdq zmm26,zmm4,zmm27,0x10 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm25,zmm5,zmm28,0x10 - vpclmulqdq zmm26,zmm6,zmm29,0x10 + vpclmulqdq zmm26,zmm4,zmm27,0x10 ; MI_4 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{4,3,2,1,0}) + vpclmulqdq zmm25,zmm5,zmm28,0x10 ; MI_5 + vpclmulqdq zmm26,zmm6,zmm29,0x10 ; MI_6 vaesenc zmm0,zmm0,zmm19 vaesenc zmm1,zmm1,zmm19 vaesenc zmm2,zmm2,zmm19 vaesenc zmm3,zmm3,zmm19 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm26,zmm31,zmm10,0x01 - vpclmulqdq zmm25,zmm7,zmm30,0x10 - vpxord zmm24,zmm24,zmm25 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq zmm26,zmm31,zmm10,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpclmulqdq zmm25,zmm7,zmm30,0x10 ; MI_7 + vpxord zmm24,zmm24,zmm25 ; MI = sum(MI_{7,6,5,4,3,2,1,0}) vaesenc zmm0,zmm0,zmm20 vaesenc zmm1,zmm1,zmm20 vaesenc zmm2,zmm2,zmm20 vaesenc zmm3,zmm3,zmm20 - vpshufd zmm10,zmm10,0x4e - vpclmulqdq zmm4,zmm4,zmm27,0x11 - vpclmulqdq zmm5,zmm5,zmm28,0x11 - vpclmulqdq zmm6,zmm6,zmm29,0x11 + vpshufd zmm10,zmm10,0x4e ; Swap halves of LO + vpclmulqdq zmm4,zmm4,zmm27,0x11 ; HI_0 + vpclmulqdq zmm5,zmm5,zmm28,0x11 ; HI_1 + vpclmulqdq zmm6,zmm6,zmm29,0x11 ; HI_2 vaesenc zmm0,zmm0,zmm21 vaesenc zmm1,zmm1,zmm21 vaesenc zmm2,zmm2,zmm21 vaesenc zmm3,zmm3,zmm21 - vpternlogd zmm24,zmm10,zmm26,0x96 - vpclmulqdq zmm7,zmm7,zmm30,0x11 - vpternlogd zmm4,zmm5,zmm6,0x96 - vpclmulqdq zmm25,zmm31,zmm24,0x01 + vpternlogd zmm24,zmm10,zmm26,0x96 ; Fold LO into MI + vpclmulqdq zmm7,zmm7,zmm30,0x11 ; HI_3 + vpternlogd zmm4,zmm5,zmm6,0x96 ; sum(HI_{2,1,0}) + vpclmulqdq zmm25,zmm31,zmm24,0x01 ; MI_L*(x^63 + x^62 + x^57) vaesenc zmm0,zmm0,zmm22 vaesenc zmm1,zmm1,zmm22 vaesenc zmm2,zmm2,zmm22 vaesenc zmm3,zmm3,zmm22 - vpxord zmm10,zmm4,zmm7 - vpshufd zmm24,zmm24,0x4e - vpternlogd zmm10,zmm24,zmm25,0x96 + vpxord zmm10,zmm4,zmm7 ; HI = sum(HI_{3,2,1,0}) + vpshufd zmm24,zmm24,0x4e ; Swap halves of MI + vpternlogd zmm10,zmm24,zmm25,0x96 ; Fold MI into HI vaesenc zmm0,zmm0,zmm23 vaesenc zmm1,zmm1,zmm23 @@ -705,36 +705,36 @@ vpshufb zmm5,zmm5,zmm8 vpshufb zmm6,zmm6,zmm8 vpshufb zmm7,zmm7,zmm8 - vpclmulqdq zmm10,zmm4,zmm27,0x00 - vpclmulqdq zmm24,zmm5,zmm28,0x00 - vpclmulqdq zmm25,zmm6,zmm29,0x00 - vpxord zmm10,zmm10,zmm24 - vpclmulqdq zmm26,zmm7,zmm30,0x00 - vpternlogd zmm10,zmm25,zmm26,0x96 - vpclmulqdq zmm24,zmm4,zmm27,0x01 - vpclmulqdq zmm25,zmm5,zmm28,0x01 - vpclmulqdq zmm26,zmm6,zmm29,0x01 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm25,zmm7,zmm30,0x01 - vpclmulqdq zmm26,zmm4,zmm27,0x10 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm25,zmm5,zmm28,0x10 - vpclmulqdq zmm26,zmm6,zmm29,0x10 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm26,zmm31,zmm10,0x01 - vpclmulqdq zmm25,zmm7,zmm30,0x10 - vpxord zmm24,zmm24,zmm25 - vpshufd zmm10,zmm10,0x4e - vpclmulqdq zmm4,zmm4,zmm27,0x11 - vpclmulqdq zmm5,zmm5,zmm28,0x11 - vpclmulqdq zmm6,zmm6,zmm29,0x11 - vpternlogd zmm24,zmm10,zmm26,0x96 - vpclmulqdq zmm7,zmm7,zmm30,0x11 - vpternlogd zmm4,zmm5,zmm6,0x96 - vpclmulqdq zmm25,zmm31,zmm24,0x01 - vpxord zmm10,zmm4,zmm7 - vpshufd zmm24,zmm24,0x4e - vpternlogd zmm10,zmm24,zmm25,0x96 + vpclmulqdq zmm10,zmm4,zmm27,0x00 ; LO_0 + vpclmulqdq zmm24,zmm5,zmm28,0x00 ; LO_1 + vpclmulqdq zmm25,zmm6,zmm29,0x00 ; LO_2 + vpxord zmm10,zmm10,zmm24 ; sum(LO_{1,0}) + vpclmulqdq zmm26,zmm7,zmm30,0x00 ; LO_3 + vpternlogd zmm10,zmm25,zmm26,0x96 ; LO = sum(LO_{3,2,1,0}) + vpclmulqdq zmm24,zmm4,zmm27,0x01 ; MI_0 + vpclmulqdq zmm25,zmm5,zmm28,0x01 ; MI_1 + vpclmulqdq zmm26,zmm6,zmm29,0x01 ; MI_2 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{2,1,0}) + vpclmulqdq zmm25,zmm7,zmm30,0x01 ; MI_3 + vpclmulqdq zmm26,zmm4,zmm27,0x10 ; MI_4 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{4,3,2,1,0}) + vpclmulqdq zmm25,zmm5,zmm28,0x10 ; MI_5 + vpclmulqdq zmm26,zmm6,zmm29,0x10 ; MI_6 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq zmm26,zmm31,zmm10,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpclmulqdq zmm25,zmm7,zmm30,0x10 ; MI_7 + vpxord zmm24,zmm24,zmm25 ; MI = sum(MI_{7,6,5,4,3,2,1,0}) + vpshufd zmm10,zmm10,0x4e ; Swap halves of LO + vpclmulqdq zmm4,zmm4,zmm27,0x11 ; HI_0 + vpclmulqdq zmm5,zmm5,zmm28,0x11 ; HI_1 + vpclmulqdq zmm6,zmm6,zmm29,0x11 ; HI_2 + vpternlogd zmm24,zmm10,zmm26,0x96 ; Fold LO into MI + vpclmulqdq zmm7,zmm7,zmm30,0x11 ; HI_3 + vpternlogd zmm4,zmm5,zmm6,0x96 ; sum(HI_{2,1,0}) + vpclmulqdq zmm25,zmm31,zmm24,0x01 ; MI_L*(x^63 + x^62 + x^57) + vpxord zmm10,zmm4,zmm7 ; HI = sum(HI_{3,2,1,0}) + vpshufd zmm24,zmm24,0x4e ; Swap halves of MI + vpternlogd zmm10,zmm24,zmm25,0x96 ; Fold MI into HI vextracti32x4 xmm4,zmm10,1 vextracti32x4 xmm5,zmm10,2 vextracti32x4 xmm6,zmm10,3 @@ -742,32 +742,32 @@ vpternlogd xmm10,xmm6,xmm5,0x96 $L$crypt_loop_4x_done__func1: - +; Check whether any data remains. test r8,r8 jz NEAR $L$done__func1 +; The data length isn't a multiple of 256 bytes. Process the remaining +; data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. +; Going one vector at a time may seem inefficient compared to having +; separate code paths for each possible number of vectors remaining. +; However, using a loop keeps the code size down, and it performs +; surprising well; modern CPUs will start executing the next iteration +; before the previous one finishes and also predict the number of loop +; iterations. For a similar reason, we roll up the AES rounds. +; +; On the last iteration, the remaining length may be less than 64 bytes. +; Handle this using masking. +; +; Since there are enough key powers available for all remaining data, +; there is no need to do a GHASH reduction after each iteration. +; Instead, multiply each remaining block by its own key power, and only +; do a GHASH reduction at the very end. - - - - - - - - - - - - - - - - - - +; Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +; is the number of blocks that remain. mov rax,r8 neg rax - and rax,-16 + and rax,-16 ; -round_up(DATALEN, 16) lea rsi,[256+rax*1+rdi] vpxor xmm4,xmm4,xmm4 vpxor xmm5,xmm5,xmm5 @@ -777,9 +777,9 @@ jb NEAR $L$partial_vec__func1 $L$crypt_loop_1x__func1: +; Process a full 64-byte vector. - - +; Encrypt a vector of counter blocks. vpshufb zmm0,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 vpxord zmm0,zmm0,zmm13 @@ -792,19 +792,19 @@ jne NEAR $L$vaesenc_loop_tail_full_vec__func1 vaesenclast zmm0,zmm0,zmm14 - +; XOR the data with the vector of keystream blocks. vmovdqu8 zmm1,ZMMWORD[rcx] vpxord zmm0,zmm0,zmm1 vmovdqu8 ZMMWORD[rdx],zmm0 - +; Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 zmm30,ZMMWORD[rsi] vpshufb zmm0,zmm0,zmm8 vpxord zmm0,zmm0,zmm10 - vpclmulqdq zmm7,zmm0,zmm30,0x00 - vpclmulqdq zmm1,zmm0,zmm30,0x01 - vpclmulqdq zmm2,zmm0,zmm30,0x10 - vpclmulqdq zmm3,zmm0,zmm30,0x11 + vpclmulqdq zmm7,zmm0,zmm30,0x00 ; a_L * b_L + vpclmulqdq zmm1,zmm0,zmm30,0x01 ; a_L * b_H + vpclmulqdq zmm2,zmm0,zmm30,0x10 ; a_H * b_L + vpclmulqdq zmm3,zmm0,zmm30,0x11 ; a_H * b_H vpxord zmm4,zmm4,zmm7 vpternlogd zmm5,zmm1,zmm2,0x96 vpxord zmm6,zmm6,zmm3 @@ -822,10 +822,10 @@ jz NEAR $L$reduce__func1 $L$partial_vec__func1: +; Process a partial vector of length 1 <= DATALEN < 64. - - - +; Set the data mask %k1 to DATALEN 1's. +; Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. mov rax,-1 bzhi rax,rax,r8 kmovq k1,rax @@ -835,8 +835,8 @@ bzhi rax,rax,r8 kmovq k2,rax - - +; Encrypt one last vector of counter blocks. This does not need to be +; masked. The counter does not need to be incremented here. vpshufb zmm0,zmm12,zmm8 vpxord zmm0,zmm0,zmm13 lea rax,[16+r9] @@ -848,38 +848,38 @@ jne NEAR $L$vaesenc_loop_tail_partialvec__func1 vaesenclast zmm0,zmm0,zmm14 - +; XOR the data with the appropriate number of keystream bytes. vmovdqu8 zmm1{k1}{z},[rcx] vpxord zmm0,zmm0,zmm1 vmovdqu8 ZMMWORD[rdx]{k1},zmm0 - - - - - - - - - - - - +; Update GHASH with the ciphertext block(s), without reducing. +; +; In the case of DATALEN < 64, the ciphertext is zero-padded to 64 +; bytes. (If decrypting, it's done by the above masked load. If +; encrypting, it's done by the below masked register-to-register move.) +; Note that if DATALEN <= 48, there will be additional padding beyond +; the padding of the last block specified by GHASH itself; i.e., there +; may be whole block(s) that get processed by the GHASH multiplication +; and reduction instructions but should not actually be included in the +; GHASH. However, any such blocks are all-zeroes, and the values that +; they're multiplied with are also all-zeroes. Therefore they just add +; 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 zmm30{k2}{z},[rsi] vmovdqu8 zmm1{k1}{z},zmm0 vpshufb zmm0,zmm1,zmm8 vpxord zmm0,zmm0,zmm10 - vpclmulqdq zmm7,zmm0,zmm30,0x00 - vpclmulqdq zmm1,zmm0,zmm30,0x01 - vpclmulqdq zmm2,zmm0,zmm30,0x10 - vpclmulqdq zmm3,zmm0,zmm30,0x11 + vpclmulqdq zmm7,zmm0,zmm30,0x00 ; a_L * b_L + vpclmulqdq zmm1,zmm0,zmm30,0x01 ; a_L * b_H + vpclmulqdq zmm2,zmm0,zmm30,0x10 ; a_H * b_L + vpclmulqdq zmm3,zmm0,zmm30,0x11 ; a_H * b_H vpxord zmm4,zmm4,zmm7 vpternlogd zmm5,zmm1,zmm2,0x96 vpxord zmm6,zmm6,zmm3 $L$reduce__func1: - +; Finally, do the GHASH reduction. vpclmulqdq zmm0,zmm31,zmm4,0x01 vpshufd zmm4,zmm4,0x4e vpternlogd zmm5,zmm4,zmm0,0x96 @@ -895,11 +895,11 @@ $L$done__func1: - +; Store the updated GHASH accumulator back to memory. vpshufb xmm10,xmm10,xmm8 vmovdqu XMMWORD[r12],xmm10 - vzeroupper + vzeroupper ; This is needed after using ymm or zmm registers. vmovdqa xmm6,XMMWORD[rsp] vmovdqa xmm7,XMMWORD[16+rsp] vmovdqa xmm8,XMMWORD[32+rsp] @@ -932,9 +932,9 @@ push r12 $L$SEH_prologue_aes_gcm_dec_update_vaes_avx512_4: - mov rsi,QWORD[64+rsp] - mov rdi,QWORD[72+rsp] - mov r12,QWORD[80+rsp] + mov rsi,QWORD[64+rsp] ; arg5 + mov rdi,QWORD[72+rsp] ; arg6 + mov r12,QWORD[80+rsp] ; arg7 sub rsp,160 $L$SEH_prologue_aes_gcm_dec_update_vaes_avx512_5: vmovdqa XMMWORD[rsp],xmm6 @@ -959,46 +959,46 @@ $L$SEH_prologue_aes_gcm_dec_update_vaes_avx512_15: $L$SEH_endprologue_aes_gcm_dec_update_vaes_avx512_16: - +; Load some constants. vbroadcasti32x4 zmm8,ZMMWORD[$L$bswap_mask] vbroadcasti32x4 zmm31,ZMMWORD[$L$gfpoly] - - +; Load the GHASH accumulator and the starting counter. +; BoringSSL passes these values in big endian format. vmovdqu xmm10,XMMWORD[r12] vpshufb xmm10,xmm10,xmm8 vbroadcasti32x4 zmm12,ZMMWORD[rsi] vpshufb zmm12,zmm12,zmm8 - - +; Load the AES key length in bytes. BoringSSL stores number of rounds +; minus 1, so convert using: AESKEYLEN = 4 * aeskey->rounds - 20. mov r10d,DWORD[240+r9] lea r10d,[((-20))+r10*4] - - - +; Make RNDKEYLAST_PTR point to the last AES round key. This is the +; round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 +; respectively. Then load the zero-th and last round keys. lea r11,[96+r10*4+r9] vbroadcasti32x4 zmm13,ZMMWORD[r9] vbroadcasti32x4 zmm14,ZMMWORD[r11] - +; Finish initializing LE_CTR by adding [0, 1, 2, 3] to its low words. vpaddd zmm12,zmm12,ZMMWORD[$L$ctr_pattern] - +; Load 4 into all 128-bit lanes of LE_CTR_INC. vbroadcasti32x4 zmm11,ZMMWORD[$L$inc_4blocks] - - +; If there are at least 256 bytes of data, then continue into the loop +; that processes 256 bytes of data at a time. Otherwise skip it. cmp r8,256 jb NEAR $L$crypt_loop_4x_done__func2 - +; Load powers of the hash key. vmovdqu8 zmm27,ZMMWORD[((256-256))+rdi] vmovdqu8 zmm28,ZMMWORD[((256-192))+rdi] vmovdqu8 zmm29,ZMMWORD[((256-128))+rdi] vmovdqu8 zmm30,ZMMWORD[((256-64))+rdi] - +; Cache as many additional AES round keys as possible. vbroadcasti32x4 zmm15,ZMMWORD[((-144))+r11] vbroadcasti32x4 zmm16,ZMMWORD[((-128))+r11] vbroadcasti32x4 zmm17,ZMMWORD[((-112))+r11] @@ -1014,9 +1014,9 @@ vmovdqu8 zmm5,ZMMWORD[64+rcx] vmovdqu8 zmm6,ZMMWORD[128+rcx] vmovdqu8 zmm7,ZMMWORD[192+rcx] - - - +; Start the AES encryption of the counter blocks. +; Increment le_ctr four times to generate four vectors of little-endian +; counter blocks, swap each to big-endian, and store them in aesdata[0-3]. vpshufb zmm0,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 vpshufb zmm1,zmm12,zmm8 @@ -1026,7 +1026,7 @@ vpshufb zmm3,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 - +; AES "round zero": XOR in the zero-th round key. vpxord zmm0,zmm0,zmm13 vpxord zmm1,zmm1,zmm13 vpxord zmm2,zmm2,zmm13 @@ -1035,7 +1035,7 @@ cmp r10d,24 jl NEAR $L$aes128__func2 je NEAR $L$aes192__func2 - +; AES-256 vbroadcasti32x4 zmm9,ZMMWORD[((-208))+r11] vaesenc zmm0,zmm0,zmm9 vaesenc zmm1,zmm1,zmm9 @@ -1063,17 +1063,17 @@ $L$aes128__func2: - - - +; Prefetch the source data 512 bytes ahead into the L1 data cache, to +; improve performance when the hardware prefetcher is disabled. Assumes the +; L1 data cache line size is 64 bytes (de facto standard on x86_64). prefetcht0 [((512+0))+rcx] prefetcht0 [((512+64))+rcx] prefetcht0 [((512+128))+rcx] prefetcht0 [((512+192))+rcx] - - - +; Finish the AES encryption of the counter blocks in AESDATA[0-3], +; interleaved with the GHASH update of the ciphertext blocks in +; GHASHDATA[0-3]. vpshufb zmm4,zmm4,zmm8 vpxord zmm4,zmm4,zmm10 vpshufb zmm5,zmm5,zmm8 @@ -1085,78 +1085,78 @@ vaesenc zmm3,zmm3,zmm15 vpshufb zmm7,zmm7,zmm8 - vpclmulqdq zmm10,zmm4,zmm27,0x00 - vpclmulqdq zmm24,zmm5,zmm28,0x00 - vpclmulqdq zmm25,zmm6,zmm29,0x00 + vpclmulqdq zmm10,zmm4,zmm27,0x00 ; LO_0 + vpclmulqdq zmm24,zmm5,zmm28,0x00 ; LO_1 + vpclmulqdq zmm25,zmm6,zmm29,0x00 ; LO_2 vaesenc zmm0,zmm0,zmm16 vaesenc zmm1,zmm1,zmm16 vaesenc zmm2,zmm2,zmm16 vaesenc zmm3,zmm3,zmm16 - vpxord zmm10,zmm10,zmm24 - vpclmulqdq zmm26,zmm7,zmm30,0x00 - vpternlogd zmm10,zmm25,zmm26,0x96 - vpclmulqdq zmm24,zmm4,zmm27,0x01 + vpxord zmm10,zmm10,zmm24 ; sum(LO_{1,0}) + vpclmulqdq zmm26,zmm7,zmm30,0x00 ; LO_3 + vpternlogd zmm10,zmm25,zmm26,0x96 ; LO = sum(LO_{3,2,1,0}) + vpclmulqdq zmm24,zmm4,zmm27,0x01 ; MI_0 vaesenc zmm0,zmm0,zmm17 vaesenc zmm1,zmm1,zmm17 vaesenc zmm2,zmm2,zmm17 vaesenc zmm3,zmm3,zmm17 - vpclmulqdq zmm25,zmm5,zmm28,0x01 - vpclmulqdq zmm26,zmm6,zmm29,0x01 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm25,zmm7,zmm30,0x01 + vpclmulqdq zmm25,zmm5,zmm28,0x01 ; MI_1 + vpclmulqdq zmm26,zmm6,zmm29,0x01 ; MI_2 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{2,1,0}) + vpclmulqdq zmm25,zmm7,zmm30,0x01 ; MI_3 vaesenc zmm0,zmm0,zmm18 vaesenc zmm1,zmm1,zmm18 vaesenc zmm2,zmm2,zmm18 vaesenc zmm3,zmm3,zmm18 - vpclmulqdq zmm26,zmm4,zmm27,0x10 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm25,zmm5,zmm28,0x10 - vpclmulqdq zmm26,zmm6,zmm29,0x10 + vpclmulqdq zmm26,zmm4,zmm27,0x10 ; MI_4 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{4,3,2,1,0}) + vpclmulqdq zmm25,zmm5,zmm28,0x10 ; MI_5 + vpclmulqdq zmm26,zmm6,zmm29,0x10 ; MI_6 vaesenc zmm0,zmm0,zmm19 vaesenc zmm1,zmm1,zmm19 vaesenc zmm2,zmm2,zmm19 vaesenc zmm3,zmm3,zmm19 - vpternlogd zmm24,zmm25,zmm26,0x96 - vpclmulqdq zmm26,zmm31,zmm10,0x01 - vpclmulqdq zmm25,zmm7,zmm30,0x10 - vpxord zmm24,zmm24,zmm25 + vpternlogd zmm24,zmm25,zmm26,0x96 ; sum(MI_{6,5,4,3,2,1,0}) + vpclmulqdq zmm26,zmm31,zmm10,0x01 ; LO_L*(x^63 + x^62 + x^57) + vpclmulqdq zmm25,zmm7,zmm30,0x10 ; MI_7 + vpxord zmm24,zmm24,zmm25 ; MI = sum(MI_{7,6,5,4,3,2,1,0}) vaesenc zmm0,zmm0,zmm20 vaesenc zmm1,zmm1,zmm20 vaesenc zmm2,zmm2,zmm20 vaesenc zmm3,zmm3,zmm20 - vpshufd zmm10,zmm10,0x4e - vpclmulqdq zmm4,zmm4,zmm27,0x11 - vpclmulqdq zmm5,zmm5,zmm28,0x11 - vpclmulqdq zmm6,zmm6,zmm29,0x11 + vpshufd zmm10,zmm10,0x4e ; Swap halves of LO + vpclmulqdq zmm4,zmm4,zmm27,0x11 ; HI_0 + vpclmulqdq zmm5,zmm5,zmm28,0x11 ; HI_1 + vpclmulqdq zmm6,zmm6,zmm29,0x11 ; HI_2 vaesenc zmm0,zmm0,zmm21 vaesenc zmm1,zmm1,zmm21 vaesenc zmm2,zmm2,zmm21 vaesenc zmm3,zmm3,zmm21 - vpternlogd zmm24,zmm10,zmm26,0x96 - vpclmulqdq zmm7,zmm7,zmm30,0x11 - vpternlogd zmm4,zmm5,zmm6,0x96 - vpclmulqdq zmm25,zmm31,zmm24,0x01 + vpternlogd zmm24,zmm10,zmm26,0x96 ; Fold LO into MI + vpclmulqdq zmm7,zmm7,zmm30,0x11 ; HI_3 + vpternlogd zmm4,zmm5,zmm6,0x96 ; sum(HI_{2,1,0}) + vpclmulqdq zmm25,zmm31,zmm24,0x01 ; MI_L*(x^63 + x^62 + x^57) vaesenc zmm0,zmm0,zmm22 vaesenc zmm1,zmm1,zmm22 vaesenc zmm2,zmm2,zmm22 vaesenc zmm3,zmm3,zmm22 - vpxord zmm10,zmm4,zmm7 - vpshufd zmm24,zmm24,0x4e - vpternlogd zmm10,zmm24,zmm25,0x96 + vpxord zmm10,zmm4,zmm7 ; HI = sum(HI_{3,2,1,0}) + vpshufd zmm24,zmm24,0x4e ; Swap halves of MI + vpternlogd zmm10,zmm24,zmm25,0x96 ; Fold MI into HI vaesenc zmm0,zmm0,zmm23 vaesenc zmm1,zmm1,zmm23 @@ -1189,32 +1189,32 @@ cmp r8,256 jae NEAR $L$crypt_loop_4x__func2 $L$crypt_loop_4x_done__func2: - +; Check whether any data remains. test r8,r8 jz NEAR $L$done__func2 +; The data length isn't a multiple of 256 bytes. Process the remaining +; data of length 1 <= DATALEN < 256, up to one 64-byte vector at a time. +; Going one vector at a time may seem inefficient compared to having +; separate code paths for each possible number of vectors remaining. +; However, using a loop keeps the code size down, and it performs +; surprising well; modern CPUs will start executing the next iteration +; before the previous one finishes and also predict the number of loop +; iterations. For a similar reason, we roll up the AES rounds. +; +; On the last iteration, the remaining length may be less than 64 bytes. +; Handle this using masking. +; +; Since there are enough key powers available for all remaining data, +; there is no need to do a GHASH reduction after each iteration. +; Instead, multiply each remaining block by its own key power, and only +; do a GHASH reduction at the very end. - - - - - - - - - - - - - - - - - - +; Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N +; is the number of blocks that remain. mov rax,r8 neg rax - and rax,-16 + and rax,-16 ; -round_up(DATALEN, 16) lea rsi,[256+rax*1+rdi] vpxor xmm4,xmm4,xmm4 vpxor xmm5,xmm5,xmm5 @@ -1224,9 +1224,9 @@ jb NEAR $L$partial_vec__func2 $L$crypt_loop_1x__func2: +; Process a full 64-byte vector. - - +; Encrypt a vector of counter blocks. vpshufb zmm0,zmm12,zmm8 vpaddd zmm12,zmm12,zmm11 vpxord zmm0,zmm0,zmm13 @@ -1239,19 +1239,19 @@ jne NEAR $L$vaesenc_loop_tail_full_vec__func2 vaesenclast zmm0,zmm0,zmm14 - +; XOR the data with the vector of keystream blocks. vmovdqu8 zmm1,ZMMWORD[rcx] vpxord zmm0,zmm0,zmm1 vmovdqu8 ZMMWORD[rdx],zmm0 - +; Update GHASH with the ciphertext blocks, without reducing. vmovdqu8 zmm30,ZMMWORD[rsi] vpshufb zmm0,zmm1,zmm8 vpxord zmm0,zmm0,zmm10 - vpclmulqdq zmm7,zmm0,zmm30,0x00 - vpclmulqdq zmm1,zmm0,zmm30,0x01 - vpclmulqdq zmm2,zmm0,zmm30,0x10 - vpclmulqdq zmm3,zmm0,zmm30,0x11 + vpclmulqdq zmm7,zmm0,zmm30,0x00 ; a_L * b_L + vpclmulqdq zmm1,zmm0,zmm30,0x01 ; a_L * b_H + vpclmulqdq zmm2,zmm0,zmm30,0x10 ; a_H * b_L + vpclmulqdq zmm3,zmm0,zmm30,0x11 ; a_H * b_H vpxord zmm4,zmm4,zmm7 vpternlogd zmm5,zmm1,zmm2,0x96 vpxord zmm6,zmm6,zmm3 @@ -1269,10 +1269,10 @@ jz NEAR $L$reduce__func2 $L$partial_vec__func2: +; Process a partial vector of length 1 <= DATALEN < 64. - - - +; Set the data mask %k1 to DATALEN 1's. +; Set the key powers mask %k2 to round_up(DATALEN, 16) 1's. mov rax,-1 bzhi rax,rax,r8 kmovq k1,rax @@ -1282,8 +1282,8 @@ bzhi rax,rax,r8 kmovq k2,rax - - +; Encrypt one last vector of counter blocks. This does not need to be +; masked. The counter does not need to be incremented here. vpshufb zmm0,zmm12,zmm8 vpxord zmm0,zmm0,zmm13 lea rax,[16+r9] @@ -1295,38 +1295,38 @@ jne NEAR $L$vaesenc_loop_tail_partialvec__func2 vaesenclast zmm0,zmm0,zmm14 - +; XOR the data with the appropriate number of keystream bytes. vmovdqu8 zmm1{k1}{z},[rcx] vpxord zmm0,zmm0,zmm1 vmovdqu8 ZMMWORD[rdx]{k1},zmm0 - - - - - - - - - - - - +; Update GHASH with the ciphertext block(s), without reducing. +; +; In the case of DATALEN < 64, the ciphertext is zero-padded to 64 +; bytes. (If decrypting, it's done by the above masked load. If +; encrypting, it's done by the below masked register-to-register move.) +; Note that if DATALEN <= 48, there will be additional padding beyond +; the padding of the last block specified by GHASH itself; i.e., there +; may be whole block(s) that get processed by the GHASH multiplication +; and reduction instructions but should not actually be included in the +; GHASH. However, any such blocks are all-zeroes, and the values that +; they're multiplied with are also all-zeroes. Therefore they just add +; 0 * 0 = 0 to the final GHASH result, which makes no difference. vmovdqu8 zmm30{k2}{z},[rsi] vpshufb zmm0,zmm1,zmm8 vpxord zmm0,zmm0,zmm10 - vpclmulqdq zmm7,zmm0,zmm30,0x00 - vpclmulqdq zmm1,zmm0,zmm30,0x01 - vpclmulqdq zmm2,zmm0,zmm30,0x10 - vpclmulqdq zmm3,zmm0,zmm30,0x11 + vpclmulqdq zmm7,zmm0,zmm30,0x00 ; a_L * b_L + vpclmulqdq zmm1,zmm0,zmm30,0x01 ; a_L * b_H + vpclmulqdq zmm2,zmm0,zmm30,0x10 ; a_H * b_L + vpclmulqdq zmm3,zmm0,zmm30,0x11 ; a_H * b_H vpxord zmm4,zmm4,zmm7 vpternlogd zmm5,zmm1,zmm2,0x96 vpxord zmm6,zmm6,zmm3 $L$reduce__func2: - +; Finally, do the GHASH reduction. vpclmulqdq zmm0,zmm31,zmm4,0x01 vpshufd zmm4,zmm4,0x4e vpternlogd zmm5,zmm4,zmm0,0x96 @@ -1342,11 +1342,11 @@ $L$done__func2: - +; Store the updated GHASH accumulator back to memory. vpshufb xmm10,xmm10,xmm8 vmovdqu XMMWORD[r12],xmm10 - vzeroupper + vzeroupper ; This is needed after using ymm or zmm registers. vmovdqa xmm6,XMMWORD[rsp] vmovdqa xmm7,XMMWORD[16+rsp] vmovdqa xmm8,XMMWORD[32+rsp] @@ -1387,7 +1387,7 @@ section .xdata rdata align=8 ALIGN 4 $L$SEH_info_gcm_gmult_vpclmulqdq_avx512_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_gmult_vpclmulqdq_avx512_4-$L$SEH_begin_gcm_gmult_vpclmulqdq_avx512_1 DB 3 DB 0 @@ -1399,7 +1399,7 @@ DW 0 $L$SEH_info_gcm_ghash_vpclmulqdq_avx512_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_ghash_vpclmulqdq_avx512_11-$L$SEH_begin_gcm_ghash_vpclmulqdq_avx512_1 DB 18 DB 0 @@ -1432,7 +1432,7 @@ DW 17 $L$SEH_info_aes_gcm_enc_update_vaes_avx512_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aes_gcm_enc_update_vaes_avx512_16-$L$SEH_begin_aes_gcm_enc_update_vaes_avx512_1 DB 25 DB 0 @@ -1478,7 +1478,7 @@ DW 0 $L$SEH_info_aes_gcm_dec_update_vaes_avx512_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aes_gcm_dec_update_vaes_avx512_16-$L$SEH_begin_aes_gcm_dec_update_vaes_avx512_1 DB 25 DB 0
diff --git a/gen/bcm/aesni-gcm-x86_64-apple.S b/gen/bcm/aesni-gcm-x86_64-apple.S index d4a9af1..aa92f4f 100644 --- a/gen/bcm/aesni-gcm-x86_64-apple.S +++ b/gen/bcm/aesni-gcm-x86_64-apple.S
@@ -10,9 +10,9 @@ .p2align 5 _aesni_ctr32_ghash_6x: - vmovdqu 32(%r11),%xmm2 + vmovdqu 32(%r11),%xmm2 // borrow %xmm2, .Lone_msb subq $6,%rdx - vpxor %xmm4,%xmm4,%xmm4 + vpxor %xmm4,%xmm4,%xmm4 // %xmm4 = 0 vmovdqu 0-128(%rcx),%xmm15 vpaddb %xmm2,%xmm1,%xmm10 vpaddb %xmm2,%xmm10,%xmm11 @@ -20,46 +20,46 @@ vpaddb %xmm2,%xmm12,%xmm13 vpaddb %xmm2,%xmm13,%xmm14 vpxor %xmm15,%xmm1,%xmm9 - vmovdqu %xmm4,16+8(%rsp) + vmovdqu %xmm4,16+8(%rsp) // "%xmm7" = 0 jmp L$oop6x .p2align 5 L$oop6x: addl $100663296,%ebx - jc L$handle_ctr32 - vmovdqu 0-32(%r9),%xmm3 - vpaddb %xmm2,%xmm14,%xmm1 + jc L$handle_ctr32 // discard ? + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 + vpaddb %xmm2,%xmm14,%xmm1 // next counter value vpxor %xmm15,%xmm10,%xmm10 vpxor %xmm15,%xmm11,%xmm11 L$resume_ctr32: - vmovdqu %xmm1,(%r8) + vmovdqu %xmm1,(%r8) // save next counter value vpclmulqdq $0x10,%xmm3,%xmm7,%xmm5 vpxor %xmm15,%xmm12,%xmm12 - vmovups 16-128(%rcx),%xmm2 + vmovups 16-128(%rcx),%xmm2 // borrow %xmm2 for %xmm15 vpclmulqdq $0x01,%xmm3,%xmm7,%xmm6 - - - - - - - - - - - - - - - - +// At this point, the current block of 96 (0x60) bytes has already been +// loaded into registers. Concurrently with processing it, we want to +// load the next 96 bytes of input for the next round. Obviously, we can +// only do this if there are at least 96 more bytes of input beyond the +// input we're currently processing, or else we'd read past the end of +// the input buffer. Here, we set |%r12| to 96 if there are at least 96 +// bytes of input beyond the 96 bytes we're already processing, and we +// set |%r12| to 0 otherwise. In the case where we set |%r12| to 96, +// we'll read in the next block so that it is in registers for the next +// loop iteration. In the case where we set |%r12| to 0, we'll re-read +// the current block and then ignore what we re-read. +// +// At this point, |%r14| points to the current (already read into +// registers) block, and |%r15| points to 2*96 bytes before the end of +// the input. Thus, |%r14| > |%r15| means that we do not have the next +// 96-byte block to read in, and |%r14| <= |%r15| means we do. xorq %r12,%r12 cmpq %r14,%r15 vaesenc %xmm2,%xmm9,%xmm9 - vmovdqu 48+8(%rsp),%xmm0 + vmovdqu 48+8(%rsp),%xmm0 // I[4] vpxor %xmm15,%xmm13,%xmm13 vpclmulqdq $0x00,%xmm3,%xmm7,%xmm1 vaesenc %xmm2,%xmm10,%xmm10 @@ -67,12 +67,12 @@ setnc %r12b vpclmulqdq $0x11,%xmm3,%xmm7,%xmm7 vaesenc %xmm2,%xmm11,%xmm11 - vmovdqu 16-32(%r9),%xmm3 + vmovdqu 16-32(%r9),%xmm3 // %xmm3^2 negq %r12 vaesenc %xmm2,%xmm12,%xmm12 vpxor %xmm5,%xmm6,%xmm6 vpclmulqdq $0x00,%xmm3,%xmm0,%xmm5 - vpxor %xmm4,%xmm8,%xmm8 + vpxor %xmm4,%xmm8,%xmm8 // modulo-scheduled vaesenc %xmm2,%xmm13,%xmm13 vpxor %xmm5,%xmm1,%xmm4 andq $0x60,%r12 @@ -83,9 +83,9 @@ vpclmulqdq $0x01,%xmm3,%xmm0,%xmm2 leaq (%r14,%r12,1),%r14 vaesenc %xmm15,%xmm9,%xmm9 - vpxor 16+8(%rsp),%xmm8,%xmm8 + vpxor 16+8(%rsp),%xmm8,%xmm8 // modulo-scheduled [vpxor %xmm7,%xmm8,%xmm8] vpclmulqdq $0x11,%xmm3,%xmm0,%xmm3 - vmovdqu 64+8(%rsp),%xmm0 + vmovdqu 64+8(%rsp),%xmm0 // I[3] vaesenc %xmm15,%xmm10,%xmm10 movbeq 88(%r14),%r13 vaesenc %xmm15,%xmm11,%xmm11 @@ -94,7 +94,7 @@ movq %r13,32+8(%rsp) vaesenc %xmm15,%xmm13,%xmm13 movq %r12,40+8(%rsp) - vmovdqu 48-32(%r9),%xmm5 + vmovdqu 48-32(%r9),%xmm5 // borrow %xmm5 for %xmm3^3 vaesenc %xmm15,%xmm14,%xmm14 vmovups 48-128(%rcx),%xmm15 @@ -108,11 +108,11 @@ vpclmulqdq $0x01,%xmm5,%xmm0,%xmm3 vaesenc %xmm15,%xmm11,%xmm11 vpclmulqdq $0x11,%xmm5,%xmm0,%xmm5 - vmovdqu 80+8(%rsp),%xmm0 + vmovdqu 80+8(%rsp),%xmm0 // I[2] vaesenc %xmm15,%xmm12,%xmm12 vaesenc %xmm15,%xmm13,%xmm13 vpxor %xmm1,%xmm4,%xmm4 - vmovdqu 64-32(%r9),%xmm1 + vmovdqu 64-32(%r9),%xmm1 // borrow %xmm1 for %xmm3^4 vaesenc %xmm15,%xmm14,%xmm14 vmovups 64-128(%rcx),%xmm15 @@ -128,13 +128,13 @@ vaesenc %xmm15,%xmm11,%xmm11 movbeq 64(%r14),%r12 vpclmulqdq $0x11,%xmm1,%xmm0,%xmm1 - vmovdqu 96+8(%rsp),%xmm0 + vmovdqu 96+8(%rsp),%xmm0 // I[1] vaesenc %xmm15,%xmm12,%xmm12 movq %r13,48+8(%rsp) vaesenc %xmm15,%xmm13,%xmm13 movq %r12,56+8(%rsp) vpxor %xmm2,%xmm4,%xmm4 - vmovdqu 96-32(%r9),%xmm2 + vmovdqu 96-32(%r9),%xmm2 // borrow %xmm2 for %xmm3^5 vaesenc %xmm15,%xmm14,%xmm14 vmovups 80-128(%rcx),%xmm15 @@ -147,7 +147,7 @@ movbeq 56(%r14),%r13 vpxor %xmm1,%xmm7,%xmm7 vpclmulqdq $0x01,%xmm2,%xmm0,%xmm1 - vpxor 112+8(%rsp),%xmm8,%xmm8 + vpxor 112+8(%rsp),%xmm8,%xmm8 // accumulate I[0] vaesenc %xmm15,%xmm11,%xmm11 movbeq 48(%r14),%r12 vpclmulqdq $0x11,%xmm2,%xmm0,%xmm2 @@ -156,7 +156,7 @@ vaesenc %xmm15,%xmm13,%xmm13 movq %r12,72+8(%rsp) vpxor %xmm3,%xmm4,%xmm4 - vmovdqu 112-32(%r9),%xmm3 + vmovdqu 112-32(%r9),%xmm3 // %xmm3^6 vaesenc %xmm15,%xmm14,%xmm14 vmovups 96-128(%rcx),%xmm15 @@ -183,7 +183,7 @@ vmovups 112-128(%rcx),%xmm15 vpslldq $8,%xmm6,%xmm5 vpxor %xmm2,%xmm4,%xmm4 - vmovdqu 16(%r11),%xmm3 + vmovdqu 16(%r11),%xmm3 // .Lpoly vaesenc %xmm15,%xmm9,%xmm9 vpxor %xmm8,%xmm7,%xmm7 @@ -192,13 +192,13 @@ movbeq 24(%r14),%r13 vaesenc %xmm15,%xmm11,%xmm11 movbeq 16(%r14),%r12 - vpalignr $8,%xmm4,%xmm4,%xmm0 + vpalignr $8,%xmm4,%xmm4,%xmm0 // 1st phase vpclmulqdq $0x10,%xmm3,%xmm4,%xmm4 movq %r13,96+8(%rsp) vaesenc %xmm15,%xmm12,%xmm12 movq %r12,104+8(%rsp) vaesenc %xmm15,%xmm13,%xmm13 - vmovups 128-128(%rcx),%xmm1 + vmovups 128-128(%rcx),%xmm1 // borrow %xmm1 for %xmm15 vaesenc %xmm15,%xmm14,%xmm14 vaesenc %xmm1,%xmm9,%xmm9 @@ -215,7 +215,7 @@ vaesenc %xmm1,%xmm14,%xmm14 vmovups 160-128(%rcx),%xmm1 cmpl $11,%r10d - jb L$enc_tail + jb L$enc_tail // 128-bit key vaesenc %xmm15,%xmm9,%xmm9 vaesenc %xmm15,%xmm10,%xmm10 @@ -232,7 +232,7 @@ vmovups 176-128(%rcx),%xmm15 vaesenc %xmm1,%xmm14,%xmm14 vmovups 192-128(%rcx),%xmm1 - je L$enc_tail + je L$enc_tail // 192-bit key vaesenc %xmm15,%xmm9,%xmm9 vaesenc %xmm15,%xmm10,%xmm10 @@ -249,16 +249,16 @@ vmovups 208-128(%rcx),%xmm15 vaesenc %xmm1,%xmm14,%xmm14 vmovups 224-128(%rcx),%xmm1 - jmp L$enc_tail + jmp L$enc_tail // 256-bit key .p2align 5 L$handle_ctr32: - vmovdqu (%r11),%xmm0 - vpshufb %xmm0,%xmm1,%xmm6 - vmovdqu 48(%r11),%xmm5 - vpaddd 64(%r11),%xmm6,%xmm10 + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + vpshufb %xmm0,%xmm1,%xmm6 // byte-swap counter + vmovdqu 48(%r11),%xmm5 // borrow %xmm5, .Ltwo_lsb + vpaddd 64(%r11),%xmm6,%xmm10 // .Lone_lsb vpaddd %xmm5,%xmm6,%xmm11 - vmovdqu 0-32(%r9),%xmm3 + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 vpaddd %xmm5,%xmm10,%xmm12 vpshufb %xmm0,%xmm10,%xmm10 vpaddd %xmm5,%xmm11,%xmm13 @@ -267,17 +267,17 @@ vpaddd %xmm5,%xmm12,%xmm14 vpshufb %xmm0,%xmm12,%xmm12 vpxor %xmm15,%xmm11,%xmm11 - vpaddd %xmm5,%xmm13,%xmm1 + vpaddd %xmm5,%xmm13,%xmm1 // byte-swapped next counter value vpshufb %xmm0,%xmm13,%xmm13 vpshufb %xmm0,%xmm14,%xmm14 - vpshufb %xmm0,%xmm1,%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 // next counter value jmp L$resume_ctr32 .p2align 5 L$enc_tail: vaesenc %xmm15,%xmm9,%xmm9 - vmovdqu %xmm7,16+8(%rsp) - vpalignr $8,%xmm4,%xmm4,%xmm8 + vmovdqu %xmm7,16+8(%rsp) // postpone vpxor %xmm7,%xmm8,%xmm8 + vpalignr $8,%xmm4,%xmm4,%xmm8 // 2nd phase vaesenc %xmm15,%xmm10,%xmm10 vpclmulqdq $0x10,%xmm3,%xmm4,%xmm4 vpxor 0(%rdi),%xmm1,%xmm2 @@ -290,16 +290,16 @@ vaesenc %xmm15,%xmm14,%xmm14 vpxor 64(%rdi),%xmm1,%xmm7 vpxor 80(%rdi),%xmm1,%xmm3 - vmovdqu (%r8),%xmm1 + vmovdqu (%r8),%xmm1 // load next counter value vaesenclast %xmm2,%xmm9,%xmm9 - vmovdqu 32(%r11),%xmm2 + vmovdqu 32(%r11),%xmm2 // borrow %xmm2, .Lone_msb vaesenclast %xmm0,%xmm10,%xmm10 vpaddb %xmm2,%xmm1,%xmm0 movq %r13,112+8(%rsp) leaq 96(%rdi),%rdi - - prefetcht0 512(%rdi) +// These two prefetches were added in BoringSSL. See change that added them. + prefetcht0 512(%rdi) // We use 96-byte block so prefetch 2 lines (128 bytes) prefetcht0 576(%rdi) vaesenclast %xmm5,%xmm11,%xmm11 vpaddb %xmm2,%xmm0,%xmm5 @@ -317,24 +317,24 @@ subq $0x6,%rdx jc L$6x_done - vmovups %xmm9,-96(%rsi) + vmovups %xmm9,-96(%rsi) // save output vpxor %xmm15,%xmm1,%xmm9 vmovups %xmm10,-80(%rsi) - vmovdqa %xmm0,%xmm10 + vmovdqa %xmm0,%xmm10 // 0 latency vmovups %xmm11,-64(%rsi) - vmovdqa %xmm5,%xmm11 + vmovdqa %xmm5,%xmm11 // 0 latency vmovups %xmm12,-48(%rsi) - vmovdqa %xmm6,%xmm12 + vmovdqa %xmm6,%xmm12 // 0 latency vmovups %xmm13,-32(%rsi) - vmovdqa %xmm7,%xmm13 + vmovdqa %xmm7,%xmm13 // 0 latency vmovups %xmm14,-16(%rsi) - vmovdqa %xmm3,%xmm14 - vmovdqu 32+8(%rsp),%xmm7 + vmovdqa %xmm3,%xmm14 // 0 latency + vmovdqu 32+8(%rsp),%xmm7 // I[5] jmp L$oop6x L$6x_done: - vpxor 16+8(%rsp),%xmm8,%xmm8 - vpxor %xmm4,%xmm8,%xmm8 + vpxor 16+8(%rsp),%xmm8,%xmm8 // modulo-scheduled + vpxor %xmm4,%xmm8,%xmm8 // modulo-scheduled ret @@ -350,15 +350,15 @@ _CET_ENDBR xorq %rax,%rax - - - cmpq $0x60,%rdx +// We call |_aesni_ctr32_ghash_6x|, which requires at least 96 (0x60) +// bytes of input. + cmpq $0x60,%rdx // minimal accepted length jb L$gcm_dec_abort pushq %rbp - movq %rsp,%rbp + movq %rsp,%rbp // save stack pointer pushq %rbx @@ -378,17 +378,17 @@ vzeroupper movq 16(%rbp),%r12 - vmovdqu (%r8),%xmm1 + vmovdqu (%r8),%xmm1 // input counter value addq $-128,%rsp movl 12(%r8),%ebx leaq L$bswap_mask(%rip),%r11 - leaq -128(%rcx),%r14 - movq $0xf80,%r15 - vmovdqu (%r12),%xmm8 - andq $-128,%rsp - vmovdqu (%r11),%xmm0 - leaq 128(%rcx),%rcx - leaq 32(%r9),%r9 + leaq -128(%rcx),%r14 // borrow %r14 + movq $0xf80,%r15 // borrow %r15 + vmovdqu (%r12),%xmm8 // load Xi + andq $-128,%rsp // ensure stack alignment + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + leaq 128(%rcx),%rcx // size optimization + leaq 32(%r9),%r9 // size optimization movl 240-128(%rcx),%r10d vpshufb %xmm0,%xmm8,%xmm8 @@ -398,26 +398,26 @@ jc L$dec_no_key_aliasing cmpq $768,%r15 jnc L$dec_no_key_aliasing - subq %r15,%rsp + subq %r15,%rsp // avoid aliasing with key L$dec_no_key_aliasing: - vmovdqu 80(%rdi),%xmm7 + vmovdqu 80(%rdi),%xmm7 // I[5] movq %rdi,%r14 vmovdqu 64(%rdi),%xmm4 - - - - - - +// |_aesni_ctr32_ghash_6x| requires |%r15| to point to 2*96 (0xc0) +// bytes before the end of the input. Note, in particular, that this is +// correct even if |%rdx| is not an even multiple of 96 or 16. XXX: This +// seems to require that |%rdi| + |%rdx| >= 2*96 (0xc0); i.e. |%rdi| must +// not be near the very beginning of the address space when |%rdx| < 2*96 +// (0xc0). leaq -192(%rdi,%rdx,1),%r15 vmovdqu 48(%rdi),%xmm5 shrq $4,%rdx xorq %rax,%rax vmovdqu 32(%rdi),%xmm6 - vpshufb %xmm0,%xmm7,%xmm7 + vpshufb %xmm0,%xmm7,%xmm7 // passed to _aesni_ctr32_ghash_6x vmovdqu 16(%rdi),%xmm2 vpshufb %xmm0,%xmm4,%xmm4 vmovdqu (%rdi),%xmm3 @@ -434,18 +434,18 @@ call _aesni_ctr32_ghash_6x movq 16(%rbp),%r12 - vmovups %xmm9,-96(%rsi) + vmovups %xmm9,-96(%rsi) // save output vmovups %xmm10,-80(%rsi) vmovups %xmm11,-64(%rsi) vmovups %xmm12,-48(%rsi) vmovups %xmm13,-32(%rsi) vmovups %xmm14,-16(%rsi) - vpshufb (%r11),%xmm8,%xmm8 - vmovdqu %xmm8,(%r12) + vpshufb (%r11),%xmm8,%xmm8 // .Lbswap_mask + vmovdqu %xmm8,(%r12) // output Xi vzeroupper - leaq -40(%rbp),%rsp + leaq -40(%rbp),%rsp // restore %rsp to fixed allocation popq %r15 @@ -469,8 +469,8 @@ .alt_entry _aesni_ctr32_6x _aesni_ctr32_6x: - vmovdqu 0-128(%rcx),%xmm4 - vmovdqu 32(%r11),%xmm2 + vmovdqu 0-128(%rcx),%xmm4 // borrow %xmm4 for %xmm15 + vmovdqu 32(%r11),%xmm2 // borrow %xmm2, .Lone_msb leaq -1(%r10),%r13 vmovups 16-128(%rcx),%xmm15 leaq 32-128(%rcx),%r12 @@ -503,7 +503,7 @@ decl %r13d jnz L$oop_ctr32 - vmovdqu (%r12),%xmm3 + vmovdqu (%r12),%xmm3 // last round key vaesenc %xmm15,%xmm9,%xmm9 vpxor 0(%rdi),%xmm3,%xmm4 vaesenc %xmm15,%xmm10,%xmm10 @@ -535,9 +535,9 @@ ret .p2align 5 L$handle_ctr32_2: - vpshufb %xmm0,%xmm1,%xmm6 - vmovdqu 48(%r11),%xmm5 - vpaddd 64(%r11),%xmm6,%xmm10 + vpshufb %xmm0,%xmm1,%xmm6 // byte-swap counter + vmovdqu 48(%r11),%xmm5 // borrow %xmm5, .Ltwo_lsb + vpaddd 64(%r11),%xmm6,%xmm10 // .Lone_lsb vpaddd %xmm5,%xmm6,%xmm11 vpaddd %xmm5,%xmm10,%xmm12 vpshufb %xmm0,%xmm10,%xmm10 @@ -547,12 +547,12 @@ vpaddd %xmm5,%xmm12,%xmm14 vpshufb %xmm0,%xmm12,%xmm12 vpxor %xmm4,%xmm11,%xmm11 - vpaddd %xmm5,%xmm13,%xmm1 + vpaddd %xmm5,%xmm13,%xmm1 // byte-swapped next counter value vpshufb %xmm0,%xmm13,%xmm13 vpxor %xmm4,%xmm12,%xmm12 vpshufb %xmm0,%xmm14,%xmm14 vpxor %xmm4,%xmm13,%xmm13 - vpshufb %xmm0,%xmm1,%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 // next counter value vpxor %xmm4,%xmm14,%xmm14 jmp L$oop_ctr32 @@ -573,16 +573,16 @@ #endif xorq %rax,%rax - - - - cmpq $288,%rdx +// We call |_aesni_ctr32_6x| twice, each call consuming 96 bytes of +// input. Then we call |_aesni_ctr32_ghash_6x|, which requires at +// least 96 more bytes of input. + cmpq $288,%rdx // minimal accepted length jb L$gcm_enc_abort pushq %rbp - movq %rsp,%rbp + movq %rsp,%rbp // save stack pointer pushq %rbx @@ -601,15 +601,15 @@ vzeroupper - vmovdqu (%r8),%xmm1 + vmovdqu (%r8),%xmm1 // input counter value addq $-128,%rsp movl 12(%r8),%ebx leaq L$bswap_mask(%rip),%r11 - leaq -128(%rcx),%r14 - movq $0xf80,%r15 - leaq 128(%rcx),%rcx - vmovdqu (%r11),%xmm0 - andq $-128,%rsp + leaq -128(%rcx),%r14 // borrow %r14 + movq $0xf80,%r15 // borrow %r15 + leaq 128(%rcx),%rcx // size optimization + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + andq $-128,%rsp // ensure stack alignment movl 240-128(%rcx),%r10d andq %r15,%r14 @@ -618,24 +618,24 @@ jc L$enc_no_key_aliasing cmpq $768,%r15 jnc L$enc_no_key_aliasing - subq %r15,%rsp + subq %r15,%rsp // avoid aliasing with key L$enc_no_key_aliasing: movq %rsi,%r14 - - - - - - - +// |_aesni_ctr32_ghash_6x| requires |%r15| to point to 2*96 (0xc0) +// bytes before the end of the input. Note, in particular, that this is +// correct even if |%rdx| is not an even multiple of 96 or 16. Unlike in +// the decryption case, there's no caveat that |%rsi| must not be near +// the very beginning of the address space, because we know that +// |%rdx| >= 3*96 from the check above, and so we know +// |%rsi| + |%rdx| >= 2*96 (0xc0). leaq -192(%rsi,%rdx,1),%r15 shrq $4,%rdx call _aesni_ctr32_6x - vpshufb %xmm0,%xmm9,%xmm8 + vpshufb %xmm0,%xmm9,%xmm8 // save bswapped output on stack vpshufb %xmm0,%xmm10,%xmm2 vmovdqu %xmm8,112(%rsp) vpshufb %xmm0,%xmm11,%xmm4 @@ -644,26 +644,26 @@ vmovdqu %xmm4,80(%rsp) vpshufb %xmm0,%xmm13,%xmm6 vmovdqu %xmm5,64(%rsp) - vpshufb %xmm0,%xmm14,%xmm7 + vpshufb %xmm0,%xmm14,%xmm7 // passed to _aesni_ctr32_ghash_6x vmovdqu %xmm6,48(%rsp) call _aesni_ctr32_6x movq 16(%rbp),%r12 - leaq 32(%r9),%r9 - vmovdqu (%r12),%xmm8 + leaq 32(%r9),%r9 // size optimization + vmovdqu (%r12),%xmm8 // load Xi subq $12,%rdx movq $192,%rax vpshufb %xmm0,%xmm8,%xmm8 call _aesni_ctr32_ghash_6x - vmovdqu 32(%rsp),%xmm7 - vmovdqu (%r11),%xmm0 - vmovdqu 0-32(%r9),%xmm3 + vmovdqu 32(%rsp),%xmm7 // I[5] + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 vpunpckhqdq %xmm7,%xmm7,%xmm1 - vmovdqu 32-32(%r9),%xmm15 - vmovups %xmm9,-96(%rsi) - vpshufb %xmm0,%xmm9,%xmm9 + vmovdqu 32-32(%r9),%xmm15 // borrow %xmm15 for + vmovups %xmm9,-96(%rsi) // save output + vpshufb %xmm0,%xmm9,%xmm9 // but keep bswapped copy vpxor %xmm7,%xmm1,%xmm1 vmovups %xmm10,-80(%rsi) vpshufb %xmm0,%xmm10,%xmm10 @@ -675,18 +675,18 @@ vpshufb %xmm0,%xmm13,%xmm13 vmovups %xmm14,-16(%rsi) vpshufb %xmm0,%xmm14,%xmm14 - vmovdqu %xmm9,16(%rsp) - vmovdqu 48(%rsp),%xmm6 - vmovdqu 16-32(%r9),%xmm0 + vmovdqu %xmm9,16(%rsp) // free %xmm9 + vmovdqu 48(%rsp),%xmm6 // I[4] + vmovdqu 16-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^2 vpunpckhqdq %xmm6,%xmm6,%xmm2 vpclmulqdq $0x00,%xmm3,%xmm7,%xmm5 vpxor %xmm6,%xmm2,%xmm2 vpclmulqdq $0x11,%xmm3,%xmm7,%xmm7 vpclmulqdq $0x00,%xmm15,%xmm1,%xmm1 - vmovdqu 64(%rsp),%xmm9 + vmovdqu 64(%rsp),%xmm9 // I[3] vpclmulqdq $0x00,%xmm0,%xmm6,%xmm4 - vmovdqu 48-32(%r9),%xmm3 + vmovdqu 48-32(%r9),%xmm3 // %xmm3^3 vpxor %xmm5,%xmm4,%xmm4 vpunpckhqdq %xmm9,%xmm9,%xmm5 vpclmulqdq $0x11,%xmm0,%xmm6,%xmm6 @@ -696,9 +696,9 @@ vmovdqu 80-32(%r9),%xmm15 vpxor %xmm1,%xmm2,%xmm2 - vmovdqu 80(%rsp),%xmm1 + vmovdqu 80(%rsp),%xmm1 // I[2] vpclmulqdq $0x00,%xmm3,%xmm9,%xmm7 - vmovdqu 64-32(%r9),%xmm0 + vmovdqu 64-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^4 vpxor %xmm4,%xmm7,%xmm7 vpunpckhqdq %xmm1,%xmm1,%xmm4 vpclmulqdq $0x11,%xmm3,%xmm9,%xmm9 @@ -707,9 +707,9 @@ vpclmulqdq $0x00,%xmm15,%xmm5,%xmm5 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu 96(%rsp),%xmm2 + vmovdqu 96(%rsp),%xmm2 // I[1] vpclmulqdq $0x00,%xmm0,%xmm1,%xmm6 - vmovdqu 96-32(%r9),%xmm3 + vmovdqu 96-32(%r9),%xmm3 // %xmm3^5 vpxor %xmm7,%xmm6,%xmm6 vpunpckhqdq %xmm2,%xmm2,%xmm7 vpclmulqdq $0x11,%xmm0,%xmm1,%xmm1 @@ -719,9 +719,9 @@ vmovdqu 128-32(%r9),%xmm15 vpxor %xmm5,%xmm4,%xmm4 - vpxor 112(%rsp),%xmm8,%xmm8 + vpxor 112(%rsp),%xmm8,%xmm8 // accumulate I[0] vpclmulqdq $0x00,%xmm3,%xmm2,%xmm5 - vmovdqu 112-32(%r9),%xmm0 + vmovdqu 112-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^6 vpunpckhqdq %xmm8,%xmm8,%xmm9 vpxor %xmm6,%xmm5,%xmm5 vpclmulqdq $0x11,%xmm3,%xmm2,%xmm2 @@ -731,7 +731,7 @@ vpxor %xmm4,%xmm7,%xmm4 vpclmulqdq $0x00,%xmm0,%xmm8,%xmm6 - vmovdqu 0-32(%r9),%xmm3 + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 vpunpckhqdq %xmm14,%xmm14,%xmm1 vpclmulqdq $0x11,%xmm0,%xmm8,%xmm8 vpxor %xmm14,%xmm1,%xmm1 @@ -741,8 +741,8 @@ vpxor %xmm2,%xmm8,%xmm7 vpxor %xmm4,%xmm9,%xmm6 - vmovdqu 16-32(%r9),%xmm0 - vpxor %xmm5,%xmm7,%xmm9 + vmovdqu 16-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^2 + vpxor %xmm5,%xmm7,%xmm9 // aggregated Karatsuba post-processing vpclmulqdq $0x00,%xmm3,%xmm14,%xmm4 vpxor %xmm9,%xmm6,%xmm6 vpunpckhqdq %xmm13,%xmm13,%xmm2 @@ -755,25 +755,25 @@ vpxor %xmm6,%xmm7,%xmm7 vpclmulqdq $0x00,%xmm0,%xmm13,%xmm5 - vmovdqu 48-32(%r9),%xmm3 + vmovdqu 48-32(%r9),%xmm3 // %xmm3^3 vpxor %xmm4,%xmm5,%xmm5 vpunpckhqdq %xmm12,%xmm12,%xmm9 vpclmulqdq $0x11,%xmm0,%xmm13,%xmm13 vpxor %xmm12,%xmm9,%xmm9 vpxor %xmm14,%xmm13,%xmm13 - vpalignr $8,%xmm8,%xmm8,%xmm14 + vpalignr $8,%xmm8,%xmm8,%xmm14 // 1st phase vpclmulqdq $0x10,%xmm15,%xmm2,%xmm2 vmovdqu 80-32(%r9),%xmm15 vpxor %xmm1,%xmm2,%xmm2 vpclmulqdq $0x00,%xmm3,%xmm12,%xmm4 - vmovdqu 64-32(%r9),%xmm0 + vmovdqu 64-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^4 vpxor %xmm5,%xmm4,%xmm4 vpunpckhqdq %xmm11,%xmm11,%xmm1 vpclmulqdq $0x11,%xmm3,%xmm12,%xmm12 vpxor %xmm11,%xmm1,%xmm1 vpxor %xmm13,%xmm12,%xmm12 - vxorps 16(%rsp),%xmm7,%xmm7 + vxorps 16(%rsp),%xmm7,%xmm7 // accumulate %xmm9 vpclmulqdq $0x00,%xmm15,%xmm9,%xmm9 vpxor %xmm2,%xmm9,%xmm9 @@ -781,12 +781,12 @@ vxorps %xmm14,%xmm8,%xmm8 vpclmulqdq $0x00,%xmm0,%xmm11,%xmm5 - vmovdqu 96-32(%r9),%xmm3 + vmovdqu 96-32(%r9),%xmm3 // %xmm3^5 vpxor %xmm4,%xmm5,%xmm5 vpunpckhqdq %xmm10,%xmm10,%xmm2 vpclmulqdq $0x11,%xmm0,%xmm11,%xmm11 vpxor %xmm10,%xmm2,%xmm2 - vpalignr $8,%xmm8,%xmm8,%xmm14 + vpalignr $8,%xmm8,%xmm8,%xmm14 // 2nd phase vpxor %xmm12,%xmm11,%xmm11 vpclmulqdq $0x10,%xmm15,%xmm1,%xmm1 vmovdqu 128-32(%r9),%xmm15 @@ -797,7 +797,7 @@ vxorps %xmm14,%xmm8,%xmm8 vpclmulqdq $0x00,%xmm3,%xmm10,%xmm4 - vmovdqu 112-32(%r9),%xmm0 + vmovdqu 112-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^6 vpxor %xmm5,%xmm4,%xmm4 vpunpckhqdq %xmm8,%xmm8,%xmm9 vpclmulqdq $0x11,%xmm3,%xmm10,%xmm10 @@ -813,28 +813,28 @@ vpxor %xmm10,%xmm7,%xmm7 vpxor %xmm2,%xmm6,%xmm6 - vpxor %xmm5,%xmm7,%xmm4 + vpxor %xmm5,%xmm7,%xmm4 // aggregated Karatsuba post-processing vpxor %xmm4,%xmm6,%xmm6 vpslldq $8,%xmm6,%xmm1 - vmovdqu 16(%r11),%xmm3 + vmovdqu 16(%r11),%xmm3 // .Lpoly vpsrldq $8,%xmm6,%xmm6 vpxor %xmm1,%xmm5,%xmm8 vpxor %xmm6,%xmm7,%xmm7 - vpalignr $8,%xmm8,%xmm8,%xmm2 + vpalignr $8,%xmm8,%xmm8,%xmm2 // 1st phase vpclmulqdq $0x10,%xmm3,%xmm8,%xmm8 vpxor %xmm2,%xmm8,%xmm8 - vpalignr $8,%xmm8,%xmm8,%xmm2 + vpalignr $8,%xmm8,%xmm8,%xmm2 // 2nd phase vpclmulqdq $0x10,%xmm3,%xmm8,%xmm8 vpxor %xmm7,%xmm2,%xmm2 vpxor %xmm2,%xmm8,%xmm8 movq 16(%rbp),%r12 - vpshufb (%r11),%xmm8,%xmm8 - vmovdqu %xmm8,(%r12) + vpshufb (%r11),%xmm8,%xmm8 // .Lbswap_mask + vmovdqu %xmm8,(%r12) // output Xi vzeroupper - leaq -40(%rbp),%rsp + leaq -40(%rbp),%rsp // restore %rsp to fixed allocation popq %r15
diff --git a/gen/bcm/aesni-gcm-x86_64-linux.S b/gen/bcm/aesni-gcm-x86_64-linux.S index c422411..723bb2e 100644 --- a/gen/bcm/aesni-gcm-x86_64-linux.S +++ b/gen/bcm/aesni-gcm-x86_64-linux.S
@@ -10,9 +10,9 @@ .align 32 _aesni_ctr32_ghash_6x: .cfi_startproc - vmovdqu 32(%r11),%xmm2 + vmovdqu 32(%r11),%xmm2 // borrow %xmm2, .Lone_msb subq $6,%rdx - vpxor %xmm4,%xmm4,%xmm4 + vpxor %xmm4,%xmm4,%xmm4 // %xmm4 = 0 vmovdqu 0-128(%rcx),%xmm15 vpaddb %xmm2,%xmm1,%xmm10 vpaddb %xmm2,%xmm10,%xmm11 @@ -20,46 +20,46 @@ vpaddb %xmm2,%xmm12,%xmm13 vpaddb %xmm2,%xmm13,%xmm14 vpxor %xmm15,%xmm1,%xmm9 - vmovdqu %xmm4,16+8(%rsp) + vmovdqu %xmm4,16+8(%rsp) // "%xmm7" = 0 jmp .Loop6x .align 32 .Loop6x: addl $100663296,%ebx - jc .Lhandle_ctr32 - vmovdqu 0-32(%r9),%xmm3 - vpaddb %xmm2,%xmm14,%xmm1 + jc .Lhandle_ctr32 // discard ? + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 + vpaddb %xmm2,%xmm14,%xmm1 // next counter value vpxor %xmm15,%xmm10,%xmm10 vpxor %xmm15,%xmm11,%xmm11 .Lresume_ctr32: - vmovdqu %xmm1,(%r8) + vmovdqu %xmm1,(%r8) // save next counter value vpclmulqdq $0x10,%xmm3,%xmm7,%xmm5 vpxor %xmm15,%xmm12,%xmm12 - vmovups 16-128(%rcx),%xmm2 + vmovups 16-128(%rcx),%xmm2 // borrow %xmm2 for %xmm15 vpclmulqdq $0x01,%xmm3,%xmm7,%xmm6 - - - - - - - - - - - - - - - - +// At this point, the current block of 96 (0x60) bytes has already been +// loaded into registers. Concurrently with processing it, we want to +// load the next 96 bytes of input for the next round. Obviously, we can +// only do this if there are at least 96 more bytes of input beyond the +// input we're currently processing, or else we'd read past the end of +// the input buffer. Here, we set |%r12| to 96 if there are at least 96 +// bytes of input beyond the 96 bytes we're already processing, and we +// set |%r12| to 0 otherwise. In the case where we set |%r12| to 96, +// we'll read in the next block so that it is in registers for the next +// loop iteration. In the case where we set |%r12| to 0, we'll re-read +// the current block and then ignore what we re-read. +// +// At this point, |%r14| points to the current (already read into +// registers) block, and |%r15| points to 2*96 bytes before the end of +// the input. Thus, |%r14| > |%r15| means that we do not have the next +// 96-byte block to read in, and |%r14| <= |%r15| means we do. xorq %r12,%r12 cmpq %r14,%r15 vaesenc %xmm2,%xmm9,%xmm9 - vmovdqu 48+8(%rsp),%xmm0 + vmovdqu 48+8(%rsp),%xmm0 // I[4] vpxor %xmm15,%xmm13,%xmm13 vpclmulqdq $0x00,%xmm3,%xmm7,%xmm1 vaesenc %xmm2,%xmm10,%xmm10 @@ -67,12 +67,12 @@ setnc %r12b vpclmulqdq $0x11,%xmm3,%xmm7,%xmm7 vaesenc %xmm2,%xmm11,%xmm11 - vmovdqu 16-32(%r9),%xmm3 + vmovdqu 16-32(%r9),%xmm3 // %xmm3^2 negq %r12 vaesenc %xmm2,%xmm12,%xmm12 vpxor %xmm5,%xmm6,%xmm6 vpclmulqdq $0x00,%xmm3,%xmm0,%xmm5 - vpxor %xmm4,%xmm8,%xmm8 + vpxor %xmm4,%xmm8,%xmm8 // modulo-scheduled vaesenc %xmm2,%xmm13,%xmm13 vpxor %xmm5,%xmm1,%xmm4 andq $0x60,%r12 @@ -83,9 +83,9 @@ vpclmulqdq $0x01,%xmm3,%xmm0,%xmm2 leaq (%r14,%r12,1),%r14 vaesenc %xmm15,%xmm9,%xmm9 - vpxor 16+8(%rsp),%xmm8,%xmm8 + vpxor 16+8(%rsp),%xmm8,%xmm8 // modulo-scheduled [vpxor %xmm7,%xmm8,%xmm8] vpclmulqdq $0x11,%xmm3,%xmm0,%xmm3 - vmovdqu 64+8(%rsp),%xmm0 + vmovdqu 64+8(%rsp),%xmm0 // I[3] vaesenc %xmm15,%xmm10,%xmm10 movbeq 88(%r14),%r13 vaesenc %xmm15,%xmm11,%xmm11 @@ -94,7 +94,7 @@ movq %r13,32+8(%rsp) vaesenc %xmm15,%xmm13,%xmm13 movq %r12,40+8(%rsp) - vmovdqu 48-32(%r9),%xmm5 + vmovdqu 48-32(%r9),%xmm5 // borrow %xmm5 for %xmm3^3 vaesenc %xmm15,%xmm14,%xmm14 vmovups 48-128(%rcx),%xmm15 @@ -108,11 +108,11 @@ vpclmulqdq $0x01,%xmm5,%xmm0,%xmm3 vaesenc %xmm15,%xmm11,%xmm11 vpclmulqdq $0x11,%xmm5,%xmm0,%xmm5 - vmovdqu 80+8(%rsp),%xmm0 + vmovdqu 80+8(%rsp),%xmm0 // I[2] vaesenc %xmm15,%xmm12,%xmm12 vaesenc %xmm15,%xmm13,%xmm13 vpxor %xmm1,%xmm4,%xmm4 - vmovdqu 64-32(%r9),%xmm1 + vmovdqu 64-32(%r9),%xmm1 // borrow %xmm1 for %xmm3^4 vaesenc %xmm15,%xmm14,%xmm14 vmovups 64-128(%rcx),%xmm15 @@ -128,13 +128,13 @@ vaesenc %xmm15,%xmm11,%xmm11 movbeq 64(%r14),%r12 vpclmulqdq $0x11,%xmm1,%xmm0,%xmm1 - vmovdqu 96+8(%rsp),%xmm0 + vmovdqu 96+8(%rsp),%xmm0 // I[1] vaesenc %xmm15,%xmm12,%xmm12 movq %r13,48+8(%rsp) vaesenc %xmm15,%xmm13,%xmm13 movq %r12,56+8(%rsp) vpxor %xmm2,%xmm4,%xmm4 - vmovdqu 96-32(%r9),%xmm2 + vmovdqu 96-32(%r9),%xmm2 // borrow %xmm2 for %xmm3^5 vaesenc %xmm15,%xmm14,%xmm14 vmovups 80-128(%rcx),%xmm15 @@ -147,7 +147,7 @@ movbeq 56(%r14),%r13 vpxor %xmm1,%xmm7,%xmm7 vpclmulqdq $0x01,%xmm2,%xmm0,%xmm1 - vpxor 112+8(%rsp),%xmm8,%xmm8 + vpxor 112+8(%rsp),%xmm8,%xmm8 // accumulate I[0] vaesenc %xmm15,%xmm11,%xmm11 movbeq 48(%r14),%r12 vpclmulqdq $0x11,%xmm2,%xmm0,%xmm2 @@ -156,7 +156,7 @@ vaesenc %xmm15,%xmm13,%xmm13 movq %r12,72+8(%rsp) vpxor %xmm3,%xmm4,%xmm4 - vmovdqu 112-32(%r9),%xmm3 + vmovdqu 112-32(%r9),%xmm3 // %xmm3^6 vaesenc %xmm15,%xmm14,%xmm14 vmovups 96-128(%rcx),%xmm15 @@ -183,7 +183,7 @@ vmovups 112-128(%rcx),%xmm15 vpslldq $8,%xmm6,%xmm5 vpxor %xmm2,%xmm4,%xmm4 - vmovdqu 16(%r11),%xmm3 + vmovdqu 16(%r11),%xmm3 // .Lpoly vaesenc %xmm15,%xmm9,%xmm9 vpxor %xmm8,%xmm7,%xmm7 @@ -192,13 +192,13 @@ movbeq 24(%r14),%r13 vaesenc %xmm15,%xmm11,%xmm11 movbeq 16(%r14),%r12 - vpalignr $8,%xmm4,%xmm4,%xmm0 + vpalignr $8,%xmm4,%xmm4,%xmm0 // 1st phase vpclmulqdq $0x10,%xmm3,%xmm4,%xmm4 movq %r13,96+8(%rsp) vaesenc %xmm15,%xmm12,%xmm12 movq %r12,104+8(%rsp) vaesenc %xmm15,%xmm13,%xmm13 - vmovups 128-128(%rcx),%xmm1 + vmovups 128-128(%rcx),%xmm1 // borrow %xmm1 for %xmm15 vaesenc %xmm15,%xmm14,%xmm14 vaesenc %xmm1,%xmm9,%xmm9 @@ -215,7 +215,7 @@ vaesenc %xmm1,%xmm14,%xmm14 vmovups 160-128(%rcx),%xmm1 cmpl $11,%r10d - jb .Lenc_tail + jb .Lenc_tail // 128-bit key vaesenc %xmm15,%xmm9,%xmm9 vaesenc %xmm15,%xmm10,%xmm10 @@ -232,7 +232,7 @@ vmovups 176-128(%rcx),%xmm15 vaesenc %xmm1,%xmm14,%xmm14 vmovups 192-128(%rcx),%xmm1 - je .Lenc_tail + je .Lenc_tail // 192-bit key vaesenc %xmm15,%xmm9,%xmm9 vaesenc %xmm15,%xmm10,%xmm10 @@ -249,16 +249,16 @@ vmovups 208-128(%rcx),%xmm15 vaesenc %xmm1,%xmm14,%xmm14 vmovups 224-128(%rcx),%xmm1 - jmp .Lenc_tail + jmp .Lenc_tail // 256-bit key .align 32 .Lhandle_ctr32: - vmovdqu (%r11),%xmm0 - vpshufb %xmm0,%xmm1,%xmm6 - vmovdqu 48(%r11),%xmm5 - vpaddd 64(%r11),%xmm6,%xmm10 + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + vpshufb %xmm0,%xmm1,%xmm6 // byte-swap counter + vmovdqu 48(%r11),%xmm5 // borrow %xmm5, .Ltwo_lsb + vpaddd 64(%r11),%xmm6,%xmm10 // .Lone_lsb vpaddd %xmm5,%xmm6,%xmm11 - vmovdqu 0-32(%r9),%xmm3 + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 vpaddd %xmm5,%xmm10,%xmm12 vpshufb %xmm0,%xmm10,%xmm10 vpaddd %xmm5,%xmm11,%xmm13 @@ -267,17 +267,17 @@ vpaddd %xmm5,%xmm12,%xmm14 vpshufb %xmm0,%xmm12,%xmm12 vpxor %xmm15,%xmm11,%xmm11 - vpaddd %xmm5,%xmm13,%xmm1 + vpaddd %xmm5,%xmm13,%xmm1 // byte-swapped next counter value vpshufb %xmm0,%xmm13,%xmm13 vpshufb %xmm0,%xmm14,%xmm14 - vpshufb %xmm0,%xmm1,%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 // next counter value jmp .Lresume_ctr32 .align 32 .Lenc_tail: vaesenc %xmm15,%xmm9,%xmm9 - vmovdqu %xmm7,16+8(%rsp) - vpalignr $8,%xmm4,%xmm4,%xmm8 + vmovdqu %xmm7,16+8(%rsp) // postpone vpxor %xmm7,%xmm8,%xmm8 + vpalignr $8,%xmm4,%xmm4,%xmm8 // 2nd phase vaesenc %xmm15,%xmm10,%xmm10 vpclmulqdq $0x10,%xmm3,%xmm4,%xmm4 vpxor 0(%rdi),%xmm1,%xmm2 @@ -290,16 +290,16 @@ vaesenc %xmm15,%xmm14,%xmm14 vpxor 64(%rdi),%xmm1,%xmm7 vpxor 80(%rdi),%xmm1,%xmm3 - vmovdqu (%r8),%xmm1 + vmovdqu (%r8),%xmm1 // load next counter value vaesenclast %xmm2,%xmm9,%xmm9 - vmovdqu 32(%r11),%xmm2 + vmovdqu 32(%r11),%xmm2 // borrow %xmm2, .Lone_msb vaesenclast %xmm0,%xmm10,%xmm10 vpaddb %xmm2,%xmm1,%xmm0 movq %r13,112+8(%rsp) leaq 96(%rdi),%rdi - - prefetcht0 512(%rdi) +// These two prefetches were added in BoringSSL. See change that added them. + prefetcht0 512(%rdi) // We use 96-byte block so prefetch 2 lines (128 bytes) prefetcht0 576(%rdi) vaesenclast %xmm5,%xmm11,%xmm11 vpaddb %xmm2,%xmm0,%xmm5 @@ -317,24 +317,24 @@ subq $0x6,%rdx jc .L6x_done - vmovups %xmm9,-96(%rsi) + vmovups %xmm9,-96(%rsi) // save output vpxor %xmm15,%xmm1,%xmm9 vmovups %xmm10,-80(%rsi) - vmovdqa %xmm0,%xmm10 + vmovdqa %xmm0,%xmm10 // 0 latency vmovups %xmm11,-64(%rsi) - vmovdqa %xmm5,%xmm11 + vmovdqa %xmm5,%xmm11 // 0 latency vmovups %xmm12,-48(%rsi) - vmovdqa %xmm6,%xmm12 + vmovdqa %xmm6,%xmm12 // 0 latency vmovups %xmm13,-32(%rsi) - vmovdqa %xmm7,%xmm13 + vmovdqa %xmm7,%xmm13 // 0 latency vmovups %xmm14,-16(%rsi) - vmovdqa %xmm3,%xmm14 - vmovdqu 32+8(%rsp),%xmm7 + vmovdqa %xmm3,%xmm14 // 0 latency + vmovdqu 32+8(%rsp),%xmm7 // I[5] jmp .Loop6x .L6x_done: - vpxor 16+8(%rsp),%xmm8,%xmm8 - vpxor %xmm4,%xmm8,%xmm8 + vpxor 16+8(%rsp),%xmm8,%xmm8 // modulo-scheduled + vpxor %xmm4,%xmm8,%xmm8 // modulo-scheduled ret .cfi_endproc @@ -349,16 +349,16 @@ _CET_ENDBR xorq %rax,%rax - - - cmpq $0x60,%rdx +// We call |_aesni_ctr32_ghash_6x|, which requires at least 96 (0x60) +// bytes of input. + cmpq $0x60,%rdx // minimal accepted length jb .Lgcm_dec_abort pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 - movq %rsp,%rbp + movq %rsp,%rbp // save stack pointer .cfi_def_cfa_register %rbp pushq %rbx .cfi_offset %rbx,-24 @@ -378,17 +378,17 @@ vzeroupper movq 16(%rbp),%r12 - vmovdqu (%r8),%xmm1 + vmovdqu (%r8),%xmm1 // input counter value addq $-128,%rsp movl 12(%r8),%ebx leaq .Lbswap_mask(%rip),%r11 - leaq -128(%rcx),%r14 - movq $0xf80,%r15 - vmovdqu (%r12),%xmm8 - andq $-128,%rsp - vmovdqu (%r11),%xmm0 - leaq 128(%rcx),%rcx - leaq 32(%r9),%r9 + leaq -128(%rcx),%r14 // borrow %r14 + movq $0xf80,%r15 // borrow %r15 + vmovdqu (%r12),%xmm8 // load Xi + andq $-128,%rsp // ensure stack alignment + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + leaq 128(%rcx),%rcx // size optimization + leaq 32(%r9),%r9 // size optimization movl 240-128(%rcx),%r10d vpshufb %xmm0,%xmm8,%xmm8 @@ -398,26 +398,26 @@ jc .Ldec_no_key_aliasing cmpq $768,%r15 jnc .Ldec_no_key_aliasing - subq %r15,%rsp + subq %r15,%rsp // avoid aliasing with key .Ldec_no_key_aliasing: - vmovdqu 80(%rdi),%xmm7 + vmovdqu 80(%rdi),%xmm7 // I[5] movq %rdi,%r14 vmovdqu 64(%rdi),%xmm4 - - - - - - +// |_aesni_ctr32_ghash_6x| requires |%r15| to point to 2*96 (0xc0) +// bytes before the end of the input. Note, in particular, that this is +// correct even if |%rdx| is not an even multiple of 96 or 16. XXX: This +// seems to require that |%rdi| + |%rdx| >= 2*96 (0xc0); i.e. |%rdi| must +// not be near the very beginning of the address space when |%rdx| < 2*96 +// (0xc0). leaq -192(%rdi,%rdx,1),%r15 vmovdqu 48(%rdi),%xmm5 shrq $4,%rdx xorq %rax,%rax vmovdqu 32(%rdi),%xmm6 - vpshufb %xmm0,%xmm7,%xmm7 + vpshufb %xmm0,%xmm7,%xmm7 // passed to _aesni_ctr32_ghash_6x vmovdqu 16(%rdi),%xmm2 vpshufb %xmm0,%xmm4,%xmm4 vmovdqu (%rdi),%xmm3 @@ -434,18 +434,18 @@ call _aesni_ctr32_ghash_6x movq 16(%rbp),%r12 - vmovups %xmm9,-96(%rsi) + vmovups %xmm9,-96(%rsi) // save output vmovups %xmm10,-80(%rsi) vmovups %xmm11,-64(%rsi) vmovups %xmm12,-48(%rsi) vmovups %xmm13,-32(%rsi) vmovups %xmm14,-16(%rsi) - vpshufb (%r11),%xmm8,%xmm8 - vmovdqu %xmm8,(%r12) + vpshufb (%r11),%xmm8,%xmm8 // .Lbswap_mask + vmovdqu %xmm8,(%r12) // output Xi vzeroupper - leaq -40(%rbp),%rsp + leaq -40(%rbp),%rsp // restore %rsp to fixed allocation .cfi_def_cfa %rsp, 0x38 popq %r15 .cfi_adjust_cfa_offset -8 @@ -474,8 +474,8 @@ .align 32 _aesni_ctr32_6x: .cfi_startproc - vmovdqu 0-128(%rcx),%xmm4 - vmovdqu 32(%r11),%xmm2 + vmovdqu 0-128(%rcx),%xmm4 // borrow %xmm4 for %xmm15 + vmovdqu 32(%r11),%xmm2 // borrow %xmm2, .Lone_msb leaq -1(%r10),%r13 vmovups 16-128(%rcx),%xmm15 leaq 32-128(%rcx),%r12 @@ -508,7 +508,7 @@ decl %r13d jnz .Loop_ctr32 - vmovdqu (%r12),%xmm3 + vmovdqu (%r12),%xmm3 // last round key vaesenc %xmm15,%xmm9,%xmm9 vpxor 0(%rdi),%xmm3,%xmm4 vaesenc %xmm15,%xmm10,%xmm10 @@ -540,9 +540,9 @@ ret .align 32 .Lhandle_ctr32_2: - vpshufb %xmm0,%xmm1,%xmm6 - vmovdqu 48(%r11),%xmm5 - vpaddd 64(%r11),%xmm6,%xmm10 + vpshufb %xmm0,%xmm1,%xmm6 // byte-swap counter + vmovdqu 48(%r11),%xmm5 // borrow %xmm5, .Ltwo_lsb + vpaddd 64(%r11),%xmm6,%xmm10 // .Lone_lsb vpaddd %xmm5,%xmm6,%xmm11 vpaddd %xmm5,%xmm10,%xmm12 vpshufb %xmm0,%xmm10,%xmm10 @@ -552,12 +552,12 @@ vpaddd %xmm5,%xmm12,%xmm14 vpshufb %xmm0,%xmm12,%xmm12 vpxor %xmm4,%xmm11,%xmm11 - vpaddd %xmm5,%xmm13,%xmm1 + vpaddd %xmm5,%xmm13,%xmm1 // byte-swapped next counter value vpshufb %xmm0,%xmm13,%xmm13 vpxor %xmm4,%xmm12,%xmm12 vpshufb %xmm0,%xmm14,%xmm14 vpxor %xmm4,%xmm13,%xmm13 - vpshufb %xmm0,%xmm1,%xmm1 + vpshufb %xmm0,%xmm1,%xmm1 // next counter value vpxor %xmm4,%xmm14,%xmm14 jmp .Loop_ctr32 .cfi_endproc @@ -578,17 +578,17 @@ #endif xorq %rax,%rax - - - - cmpq $288,%rdx +// We call |_aesni_ctr32_6x| twice, each call consuming 96 bytes of +// input. Then we call |_aesni_ctr32_ghash_6x|, which requires at +// least 96 more bytes of input. + cmpq $288,%rdx // minimal accepted length jb .Lgcm_enc_abort pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 - movq %rsp,%rbp + movq %rsp,%rbp // save stack pointer .cfi_def_cfa_register %rbp pushq %rbx .cfi_offset %rbx,-24 @@ -607,15 +607,15 @@ vzeroupper - vmovdqu (%r8),%xmm1 + vmovdqu (%r8),%xmm1 // input counter value addq $-128,%rsp movl 12(%r8),%ebx leaq .Lbswap_mask(%rip),%r11 - leaq -128(%rcx),%r14 - movq $0xf80,%r15 - leaq 128(%rcx),%rcx - vmovdqu (%r11),%xmm0 - andq $-128,%rsp + leaq -128(%rcx),%r14 // borrow %r14 + movq $0xf80,%r15 // borrow %r15 + leaq 128(%rcx),%rcx // size optimization + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + andq $-128,%rsp // ensure stack alignment movl 240-128(%rcx),%r10d andq %r15,%r14 @@ -624,24 +624,24 @@ jc .Lenc_no_key_aliasing cmpq $768,%r15 jnc .Lenc_no_key_aliasing - subq %r15,%rsp + subq %r15,%rsp // avoid aliasing with key .Lenc_no_key_aliasing: movq %rsi,%r14 - - - - - - - +// |_aesni_ctr32_ghash_6x| requires |%r15| to point to 2*96 (0xc0) +// bytes before the end of the input. Note, in particular, that this is +// correct even if |%rdx| is not an even multiple of 96 or 16. Unlike in +// the decryption case, there's no caveat that |%rsi| must not be near +// the very beginning of the address space, because we know that +// |%rdx| >= 3*96 from the check above, and so we know +// |%rsi| + |%rdx| >= 2*96 (0xc0). leaq -192(%rsi,%rdx,1),%r15 shrq $4,%rdx call _aesni_ctr32_6x - vpshufb %xmm0,%xmm9,%xmm8 + vpshufb %xmm0,%xmm9,%xmm8 // save bswapped output on stack vpshufb %xmm0,%xmm10,%xmm2 vmovdqu %xmm8,112(%rsp) vpshufb %xmm0,%xmm11,%xmm4 @@ -650,26 +650,26 @@ vmovdqu %xmm4,80(%rsp) vpshufb %xmm0,%xmm13,%xmm6 vmovdqu %xmm5,64(%rsp) - vpshufb %xmm0,%xmm14,%xmm7 + vpshufb %xmm0,%xmm14,%xmm7 // passed to _aesni_ctr32_ghash_6x vmovdqu %xmm6,48(%rsp) call _aesni_ctr32_6x movq 16(%rbp),%r12 - leaq 32(%r9),%r9 - vmovdqu (%r12),%xmm8 + leaq 32(%r9),%r9 // size optimization + vmovdqu (%r12),%xmm8 // load Xi subq $12,%rdx movq $192,%rax vpshufb %xmm0,%xmm8,%xmm8 call _aesni_ctr32_ghash_6x - vmovdqu 32(%rsp),%xmm7 - vmovdqu (%r11),%xmm0 - vmovdqu 0-32(%r9),%xmm3 + vmovdqu 32(%rsp),%xmm7 // I[5] + vmovdqu (%r11),%xmm0 // borrow %xmm0 for .Lbswap_mask + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 vpunpckhqdq %xmm7,%xmm7,%xmm1 - vmovdqu 32-32(%r9),%xmm15 - vmovups %xmm9,-96(%rsi) - vpshufb %xmm0,%xmm9,%xmm9 + vmovdqu 32-32(%r9),%xmm15 // borrow %xmm15 for + vmovups %xmm9,-96(%rsi) // save output + vpshufb %xmm0,%xmm9,%xmm9 // but keep bswapped copy vpxor %xmm7,%xmm1,%xmm1 vmovups %xmm10,-80(%rsi) vpshufb %xmm0,%xmm10,%xmm10 @@ -681,18 +681,18 @@ vpshufb %xmm0,%xmm13,%xmm13 vmovups %xmm14,-16(%rsi) vpshufb %xmm0,%xmm14,%xmm14 - vmovdqu %xmm9,16(%rsp) - vmovdqu 48(%rsp),%xmm6 - vmovdqu 16-32(%r9),%xmm0 + vmovdqu %xmm9,16(%rsp) // free %xmm9 + vmovdqu 48(%rsp),%xmm6 // I[4] + vmovdqu 16-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^2 vpunpckhqdq %xmm6,%xmm6,%xmm2 vpclmulqdq $0x00,%xmm3,%xmm7,%xmm5 vpxor %xmm6,%xmm2,%xmm2 vpclmulqdq $0x11,%xmm3,%xmm7,%xmm7 vpclmulqdq $0x00,%xmm15,%xmm1,%xmm1 - vmovdqu 64(%rsp),%xmm9 + vmovdqu 64(%rsp),%xmm9 // I[3] vpclmulqdq $0x00,%xmm0,%xmm6,%xmm4 - vmovdqu 48-32(%r9),%xmm3 + vmovdqu 48-32(%r9),%xmm3 // %xmm3^3 vpxor %xmm5,%xmm4,%xmm4 vpunpckhqdq %xmm9,%xmm9,%xmm5 vpclmulqdq $0x11,%xmm0,%xmm6,%xmm6 @@ -702,9 +702,9 @@ vmovdqu 80-32(%r9),%xmm15 vpxor %xmm1,%xmm2,%xmm2 - vmovdqu 80(%rsp),%xmm1 + vmovdqu 80(%rsp),%xmm1 // I[2] vpclmulqdq $0x00,%xmm3,%xmm9,%xmm7 - vmovdqu 64-32(%r9),%xmm0 + vmovdqu 64-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^4 vpxor %xmm4,%xmm7,%xmm7 vpunpckhqdq %xmm1,%xmm1,%xmm4 vpclmulqdq $0x11,%xmm3,%xmm9,%xmm9 @@ -713,9 +713,9 @@ vpclmulqdq $0x00,%xmm15,%xmm5,%xmm5 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu 96(%rsp),%xmm2 + vmovdqu 96(%rsp),%xmm2 // I[1] vpclmulqdq $0x00,%xmm0,%xmm1,%xmm6 - vmovdqu 96-32(%r9),%xmm3 + vmovdqu 96-32(%r9),%xmm3 // %xmm3^5 vpxor %xmm7,%xmm6,%xmm6 vpunpckhqdq %xmm2,%xmm2,%xmm7 vpclmulqdq $0x11,%xmm0,%xmm1,%xmm1 @@ -725,9 +725,9 @@ vmovdqu 128-32(%r9),%xmm15 vpxor %xmm5,%xmm4,%xmm4 - vpxor 112(%rsp),%xmm8,%xmm8 + vpxor 112(%rsp),%xmm8,%xmm8 // accumulate I[0] vpclmulqdq $0x00,%xmm3,%xmm2,%xmm5 - vmovdqu 112-32(%r9),%xmm0 + vmovdqu 112-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^6 vpunpckhqdq %xmm8,%xmm8,%xmm9 vpxor %xmm6,%xmm5,%xmm5 vpclmulqdq $0x11,%xmm3,%xmm2,%xmm2 @@ -737,7 +737,7 @@ vpxor %xmm4,%xmm7,%xmm4 vpclmulqdq $0x00,%xmm0,%xmm8,%xmm6 - vmovdqu 0-32(%r9),%xmm3 + vmovdqu 0-32(%r9),%xmm3 // %xmm3^1 vpunpckhqdq %xmm14,%xmm14,%xmm1 vpclmulqdq $0x11,%xmm0,%xmm8,%xmm8 vpxor %xmm14,%xmm1,%xmm1 @@ -747,8 +747,8 @@ vpxor %xmm2,%xmm8,%xmm7 vpxor %xmm4,%xmm9,%xmm6 - vmovdqu 16-32(%r9),%xmm0 - vpxor %xmm5,%xmm7,%xmm9 + vmovdqu 16-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^2 + vpxor %xmm5,%xmm7,%xmm9 // aggregated Karatsuba post-processing vpclmulqdq $0x00,%xmm3,%xmm14,%xmm4 vpxor %xmm9,%xmm6,%xmm6 vpunpckhqdq %xmm13,%xmm13,%xmm2 @@ -761,25 +761,25 @@ vpxor %xmm6,%xmm7,%xmm7 vpclmulqdq $0x00,%xmm0,%xmm13,%xmm5 - vmovdqu 48-32(%r9),%xmm3 + vmovdqu 48-32(%r9),%xmm3 // %xmm3^3 vpxor %xmm4,%xmm5,%xmm5 vpunpckhqdq %xmm12,%xmm12,%xmm9 vpclmulqdq $0x11,%xmm0,%xmm13,%xmm13 vpxor %xmm12,%xmm9,%xmm9 vpxor %xmm14,%xmm13,%xmm13 - vpalignr $8,%xmm8,%xmm8,%xmm14 + vpalignr $8,%xmm8,%xmm8,%xmm14 // 1st phase vpclmulqdq $0x10,%xmm15,%xmm2,%xmm2 vmovdqu 80-32(%r9),%xmm15 vpxor %xmm1,%xmm2,%xmm2 vpclmulqdq $0x00,%xmm3,%xmm12,%xmm4 - vmovdqu 64-32(%r9),%xmm0 + vmovdqu 64-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^4 vpxor %xmm5,%xmm4,%xmm4 vpunpckhqdq %xmm11,%xmm11,%xmm1 vpclmulqdq $0x11,%xmm3,%xmm12,%xmm12 vpxor %xmm11,%xmm1,%xmm1 vpxor %xmm13,%xmm12,%xmm12 - vxorps 16(%rsp),%xmm7,%xmm7 + vxorps 16(%rsp),%xmm7,%xmm7 // accumulate %xmm9 vpclmulqdq $0x00,%xmm15,%xmm9,%xmm9 vpxor %xmm2,%xmm9,%xmm9 @@ -787,12 +787,12 @@ vxorps %xmm14,%xmm8,%xmm8 vpclmulqdq $0x00,%xmm0,%xmm11,%xmm5 - vmovdqu 96-32(%r9),%xmm3 + vmovdqu 96-32(%r9),%xmm3 // %xmm3^5 vpxor %xmm4,%xmm5,%xmm5 vpunpckhqdq %xmm10,%xmm10,%xmm2 vpclmulqdq $0x11,%xmm0,%xmm11,%xmm11 vpxor %xmm10,%xmm2,%xmm2 - vpalignr $8,%xmm8,%xmm8,%xmm14 + vpalignr $8,%xmm8,%xmm8,%xmm14 // 2nd phase vpxor %xmm12,%xmm11,%xmm11 vpclmulqdq $0x10,%xmm15,%xmm1,%xmm1 vmovdqu 128-32(%r9),%xmm15 @@ -803,7 +803,7 @@ vxorps %xmm14,%xmm8,%xmm8 vpclmulqdq $0x00,%xmm3,%xmm10,%xmm4 - vmovdqu 112-32(%r9),%xmm0 + vmovdqu 112-32(%r9),%xmm0 // borrow %xmm0 for %xmm3^6 vpxor %xmm5,%xmm4,%xmm4 vpunpckhqdq %xmm8,%xmm8,%xmm9 vpclmulqdq $0x11,%xmm3,%xmm10,%xmm10 @@ -819,28 +819,28 @@ vpxor %xmm10,%xmm7,%xmm7 vpxor %xmm2,%xmm6,%xmm6 - vpxor %xmm5,%xmm7,%xmm4 + vpxor %xmm5,%xmm7,%xmm4 // aggregated Karatsuba post-processing vpxor %xmm4,%xmm6,%xmm6 vpslldq $8,%xmm6,%xmm1 - vmovdqu 16(%r11),%xmm3 + vmovdqu 16(%r11),%xmm3 // .Lpoly vpsrldq $8,%xmm6,%xmm6 vpxor %xmm1,%xmm5,%xmm8 vpxor %xmm6,%xmm7,%xmm7 - vpalignr $8,%xmm8,%xmm8,%xmm2 + vpalignr $8,%xmm8,%xmm8,%xmm2 // 1st phase vpclmulqdq $0x10,%xmm3,%xmm8,%xmm8 vpxor %xmm2,%xmm8,%xmm8 - vpalignr $8,%xmm8,%xmm8,%xmm2 + vpalignr $8,%xmm8,%xmm8,%xmm2 // 2nd phase vpclmulqdq $0x10,%xmm3,%xmm8,%xmm8 vpxor %xmm7,%xmm2,%xmm2 vpxor %xmm2,%xmm8,%xmm8 movq 16(%rbp),%r12 - vpshufb (%r11),%xmm8,%xmm8 - vmovdqu %xmm8,(%r12) + vpshufb (%r11),%xmm8,%xmm8 // .Lbswap_mask + vmovdqu %xmm8,(%r12) // output Xi vzeroupper - leaq -40(%rbp),%rsp + leaq -40(%rbp),%rsp // restore %rsp to fixed allocation .cfi_def_cfa %rsp, 0x38 popq %r15 .cfi_adjust_cfa_offset -8
diff --git a/gen/bcm/aesni-gcm-x86_64-win.asm b/gen/bcm/aesni-gcm-x86_64-win.asm index 73a19f4..a17d4f4 100644 --- a/gen/bcm/aesni-gcm-x86_64-win.asm +++ b/gen/bcm/aesni-gcm-x86_64-win.asm
@@ -18,9 +18,9 @@ ALIGN 32 _aesni_ctr32_ghash_6x: - vmovdqu xmm2,XMMWORD[32+r11] + vmovdqu xmm2,XMMWORD[32+r11] ; borrow %xmm2, .Lone_msb sub r8,6 - vpxor xmm4,xmm4,xmm4 + vpxor xmm4,xmm4,xmm4 ; %xmm4 = 0 vmovdqu xmm15,XMMWORD[((0-128))+r9] vpaddb xmm10,xmm1,xmm2 vpaddb xmm11,xmm10,xmm2 @@ -28,46 +28,46 @@ vpaddb xmm13,xmm12,xmm2 vpaddb xmm14,xmm13,xmm2 vpxor xmm9,xmm1,xmm15 - vmovdqu XMMWORD[(16+8)+rsp],xmm4 + vmovdqu XMMWORD[(16+8)+rsp],xmm4 ; "%xmm7" = 0 jmp NEAR $L$oop6x ALIGN 32 $L$oop6x: add ebx,100663296 - jc NEAR $L$handle_ctr32 - vmovdqu xmm3,XMMWORD[((0-32))+rsi] - vpaddb xmm1,xmm14,xmm2 + jc NEAR $L$handle_ctr32 ; discard ? + vmovdqu xmm3,XMMWORD[((0-32))+rsi] ; %xmm3^1 + vpaddb xmm1,xmm14,xmm2 ; next counter value vpxor xmm10,xmm10,xmm15 vpxor xmm11,xmm11,xmm15 $L$resume_ctr32: - vmovdqu XMMWORD[rdi],xmm1 + vmovdqu XMMWORD[rdi],xmm1 ; save next counter value vpclmulqdq xmm5,xmm7,xmm3,0x10 vpxor xmm12,xmm12,xmm15 - vmovups xmm2,XMMWORD[((16-128))+r9] + vmovups xmm2,XMMWORD[((16-128))+r9] ; borrow %xmm2 for %xmm15 vpclmulqdq xmm6,xmm7,xmm3,0x01 - - - - - - - - - - - - - - - - +; At this point, the current block of 96 (0x60) bytes has already been +; loaded into registers. Concurrently with processing it, we want to +; load the next 96 bytes of input for the next round. Obviously, we can +; only do this if there are at least 96 more bytes of input beyond the +; input we're currently processing, or else we'd read past the end of +; the input buffer. Here, we set |%r12| to 96 if there are at least 96 +; bytes of input beyond the 96 bytes we're already processing, and we +; set |%r12| to 0 otherwise. In the case where we set |%r12| to 96, +; we'll read in the next block so that it is in registers for the next +; loop iteration. In the case where we set |%r12| to 0, we'll re-read +; the current block and then ignore what we re-read. +; +; At this point, |%r14| points to the current (already read into +; registers) block, and |%r15| points to 2*96 bytes before the end of +; the input. Thus, |%r14| > |%r15| means that we do not have the next +; 96-byte block to read in, and |%r14| <= |%r15| means we do. xor r12,r12 cmp r15,r14 vaesenc xmm9,xmm9,xmm2 - vmovdqu xmm0,XMMWORD[((48+8))+rsp] + vmovdqu xmm0,XMMWORD[((48+8))+rsp] ; I[4] vpxor xmm13,xmm13,xmm15 vpclmulqdq xmm1,xmm7,xmm3,0x00 vaesenc xmm10,xmm10,xmm2 @@ -75,12 +75,12 @@ setnc r12b vpclmulqdq xmm7,xmm7,xmm3,0x11 vaesenc xmm11,xmm11,xmm2 - vmovdqu xmm3,XMMWORD[((16-32))+rsi] + vmovdqu xmm3,XMMWORD[((16-32))+rsi] ; %xmm3^2 neg r12 vaesenc xmm12,xmm12,xmm2 vpxor xmm6,xmm6,xmm5 vpclmulqdq xmm5,xmm0,xmm3,0x00 - vpxor xmm8,xmm8,xmm4 + vpxor xmm8,xmm8,xmm4 ; modulo-scheduled vaesenc xmm13,xmm13,xmm2 vpxor xmm4,xmm1,xmm5 and r12,0x60 @@ -91,9 +91,9 @@ vpclmulqdq xmm2,xmm0,xmm3,0x01 lea r14,[r12*1+r14] vaesenc xmm9,xmm9,xmm15 - vpxor xmm8,xmm8,XMMWORD[((16+8))+rsp] + vpxor xmm8,xmm8,XMMWORD[((16+8))+rsp] ; modulo-scheduled [vpxor %xmm7,%xmm8,%xmm8] vpclmulqdq xmm3,xmm0,xmm3,0x11 - vmovdqu xmm0,XMMWORD[((64+8))+rsp] + vmovdqu xmm0,XMMWORD[((64+8))+rsp] ; I[3] vaesenc xmm10,xmm10,xmm15 movbe r13,QWORD[88+r14] vaesenc xmm11,xmm11,xmm15 @@ -102,7 +102,7 @@ mov QWORD[((32+8))+rsp],r13 vaesenc xmm13,xmm13,xmm15 mov QWORD[((40+8))+rsp],r12 - vmovdqu xmm5,XMMWORD[((48-32))+rsi] + vmovdqu xmm5,XMMWORD[((48-32))+rsi] ; borrow %xmm5 for %xmm3^3 vaesenc xmm14,xmm14,xmm15 vmovups xmm15,XMMWORD[((48-128))+r9] @@ -116,11 +116,11 @@ vpclmulqdq xmm3,xmm0,xmm5,0x01 vaesenc xmm11,xmm11,xmm15 vpclmulqdq xmm5,xmm0,xmm5,0x11 - vmovdqu xmm0,XMMWORD[((80+8))+rsp] + vmovdqu xmm0,XMMWORD[((80+8))+rsp] ; I[2] vaesenc xmm12,xmm12,xmm15 vaesenc xmm13,xmm13,xmm15 vpxor xmm4,xmm4,xmm1 - vmovdqu xmm1,XMMWORD[((64-32))+rsi] + vmovdqu xmm1,XMMWORD[((64-32))+rsi] ; borrow %xmm1 for %xmm3^4 vaesenc xmm14,xmm14,xmm15 vmovups xmm15,XMMWORD[((64-128))+r9] @@ -136,13 +136,13 @@ vaesenc xmm11,xmm11,xmm15 movbe r12,QWORD[64+r14] vpclmulqdq xmm1,xmm0,xmm1,0x11 - vmovdqu xmm0,XMMWORD[((96+8))+rsp] + vmovdqu xmm0,XMMWORD[((96+8))+rsp] ; I[1] vaesenc xmm12,xmm12,xmm15 mov QWORD[((48+8))+rsp],r13 vaesenc xmm13,xmm13,xmm15 mov QWORD[((56+8))+rsp],r12 vpxor xmm4,xmm4,xmm2 - vmovdqu xmm2,XMMWORD[((96-32))+rsi] + vmovdqu xmm2,XMMWORD[((96-32))+rsi] ; borrow %xmm2 for %xmm3^5 vaesenc xmm14,xmm14,xmm15 vmovups xmm15,XMMWORD[((80-128))+r9] @@ -155,7 +155,7 @@ movbe r13,QWORD[56+r14] vpxor xmm7,xmm7,xmm1 vpclmulqdq xmm1,xmm0,xmm2,0x01 - vpxor xmm8,xmm8,XMMWORD[((112+8))+rsp] + vpxor xmm8,xmm8,XMMWORD[((112+8))+rsp] ; accumulate I[0] vaesenc xmm11,xmm11,xmm15 movbe r12,QWORD[48+r14] vpclmulqdq xmm2,xmm0,xmm2,0x11 @@ -164,7 +164,7 @@ vaesenc xmm13,xmm13,xmm15 mov QWORD[((72+8))+rsp],r12 vpxor xmm4,xmm4,xmm3 - vmovdqu xmm3,XMMWORD[((112-32))+rsi] + vmovdqu xmm3,XMMWORD[((112-32))+rsi] ; %xmm3^6 vaesenc xmm14,xmm14,xmm15 vmovups xmm15,XMMWORD[((96-128))+r9] @@ -191,7 +191,7 @@ vmovups xmm15,XMMWORD[((112-128))+r9] vpslldq xmm5,xmm6,8 vpxor xmm4,xmm4,xmm2 - vmovdqu xmm3,XMMWORD[16+r11] + vmovdqu xmm3,XMMWORD[16+r11] ; .Lpoly vaesenc xmm9,xmm9,xmm15 vpxor xmm7,xmm7,xmm8 @@ -200,13 +200,13 @@ movbe r13,QWORD[24+r14] vaesenc xmm11,xmm11,xmm15 movbe r12,QWORD[16+r14] - vpalignr xmm0,xmm4,xmm4,8 + vpalignr xmm0,xmm4,xmm4,8 ; 1st phase vpclmulqdq xmm4,xmm4,xmm3,0x10 mov QWORD[((96+8))+rsp],r13 vaesenc xmm12,xmm12,xmm15 mov QWORD[((104+8))+rsp],r12 vaesenc xmm13,xmm13,xmm15 - vmovups xmm1,XMMWORD[((128-128))+r9] + vmovups xmm1,XMMWORD[((128-128))+r9] ; borrow %xmm1 for %xmm15 vaesenc xmm14,xmm14,xmm15 vaesenc xmm9,xmm9,xmm1 @@ -223,7 +223,7 @@ vaesenc xmm14,xmm14,xmm1 vmovups xmm1,XMMWORD[((160-128))+r9] cmp r10d,11 - jb NEAR $L$enc_tail + jb NEAR $L$enc_tail ; 128-bit key vaesenc xmm9,xmm9,xmm15 vaesenc xmm10,xmm10,xmm15 @@ -240,7 +240,7 @@ vmovups xmm15,XMMWORD[((176-128))+r9] vaesenc xmm14,xmm14,xmm1 vmovups xmm1,XMMWORD[((192-128))+r9] - je NEAR $L$enc_tail + je NEAR $L$enc_tail ; 192-bit key vaesenc xmm9,xmm9,xmm15 vaesenc xmm10,xmm10,xmm15 @@ -257,16 +257,16 @@ vmovups xmm15,XMMWORD[((208-128))+r9] vaesenc xmm14,xmm14,xmm1 vmovups xmm1,XMMWORD[((224-128))+r9] - jmp NEAR $L$enc_tail + jmp NEAR $L$enc_tail ; 256-bit key ALIGN 32 $L$handle_ctr32: - vmovdqu xmm0,XMMWORD[r11] - vpshufb xmm6,xmm1,xmm0 - vmovdqu xmm5,XMMWORD[48+r11] - vpaddd xmm10,xmm6,XMMWORD[64+r11] + vmovdqu xmm0,XMMWORD[r11] ; borrow %xmm0 for .Lbswap_mask + vpshufb xmm6,xmm1,xmm0 ; byte-swap counter + vmovdqu xmm5,XMMWORD[48+r11] ; borrow %xmm5, .Ltwo_lsb + vpaddd xmm10,xmm6,XMMWORD[64+r11] ; .Lone_lsb vpaddd xmm11,xmm6,xmm5 - vmovdqu xmm3,XMMWORD[((0-32))+rsi] + vmovdqu xmm3,XMMWORD[((0-32))+rsi] ; %xmm3^1 vpaddd xmm12,xmm10,xmm5 vpshufb xmm10,xmm10,xmm0 vpaddd xmm13,xmm11,xmm5 @@ -275,17 +275,17 @@ vpaddd xmm14,xmm12,xmm5 vpshufb xmm12,xmm12,xmm0 vpxor xmm11,xmm11,xmm15 - vpaddd xmm1,xmm13,xmm5 + vpaddd xmm1,xmm13,xmm5 ; byte-swapped next counter value vpshufb xmm13,xmm13,xmm0 vpshufb xmm14,xmm14,xmm0 - vpshufb xmm1,xmm1,xmm0 + vpshufb xmm1,xmm1,xmm0 ; next counter value jmp NEAR $L$resume_ctr32 ALIGN 32 $L$enc_tail: vaesenc xmm9,xmm9,xmm15 - vmovdqu XMMWORD[(16+8)+rsp],xmm7 - vpalignr xmm8,xmm4,xmm4,8 + vmovdqu XMMWORD[(16+8)+rsp],xmm7 ; postpone vpxor %xmm7,%xmm8,%xmm8 + vpalignr xmm8,xmm4,xmm4,8 ; 2nd phase vaesenc xmm10,xmm10,xmm15 vpclmulqdq xmm4,xmm4,xmm3,0x10 vpxor xmm2,xmm1,XMMWORD[rcx] @@ -298,16 +298,16 @@ vaesenc xmm14,xmm14,xmm15 vpxor xmm7,xmm1,XMMWORD[64+rcx] vpxor xmm3,xmm1,XMMWORD[80+rcx] - vmovdqu xmm1,XMMWORD[rdi] + vmovdqu xmm1,XMMWORD[rdi] ; load next counter value vaesenclast xmm9,xmm9,xmm2 - vmovdqu xmm2,XMMWORD[32+r11] + vmovdqu xmm2,XMMWORD[32+r11] ; borrow %xmm2, .Lone_msb vaesenclast xmm10,xmm10,xmm0 vpaddb xmm0,xmm1,xmm2 mov QWORD[((112+8))+rsp],r13 lea rcx,[96+rcx] - - prefetcht0 [512+rcx] +; These two prefetches were added in BoringSSL. See change that added them. + prefetcht0 [512+rcx] ; We use 96-byte block so prefetch 2 lines (128 bytes) prefetcht0 [576+rcx] vaesenclast xmm11,xmm11,xmm5 vpaddb xmm5,xmm0,xmm2 @@ -325,24 +325,24 @@ sub r8,0x6 jc NEAR $L$6x_done - vmovups XMMWORD[(-96)+rdx],xmm9 + vmovups XMMWORD[(-96)+rdx],xmm9 ; save output vpxor xmm9,xmm1,xmm15 vmovups XMMWORD[(-80)+rdx],xmm10 - vmovdqa xmm10,xmm0 + vmovdqa xmm10,xmm0 ; 0 latency vmovups XMMWORD[(-64)+rdx],xmm11 - vmovdqa xmm11,xmm5 + vmovdqa xmm11,xmm5 ; 0 latency vmovups XMMWORD[(-48)+rdx],xmm12 - vmovdqa xmm12,xmm6 + vmovdqa xmm12,xmm6 ; 0 latency vmovups XMMWORD[(-32)+rdx],xmm13 - vmovdqa xmm13,xmm7 + vmovdqa xmm13,xmm7 ; 0 latency vmovups XMMWORD[(-16)+rdx],xmm14 - vmovdqa xmm14,xmm3 - vmovdqu xmm7,XMMWORD[((32+8))+rsp] + vmovdqa xmm14,xmm3 ; 0 latency + vmovdqu xmm7,XMMWORD[((32+8))+rsp] ; I[5] jmp NEAR $L$oop6x $L$6x_done: - vpxor xmm8,xmm8,XMMWORD[((16+8))+rsp] - vpxor xmm8,xmm8,xmm4 + vpxor xmm8,xmm8,XMMWORD[((16+8))+rsp] ; modulo-scheduled + vpxor xmm8,xmm8,xmm4 ; modulo-scheduled ret @@ -356,15 +356,15 @@ _CET_ENDBR xor rax,rax - - - cmp r8,0x60 +; We call |_aesni_ctr32_ghash_6x|, which requires at least 96 (0x60) +; bytes of input. + cmp r8,0x60 ; minimal accepted length jb NEAR $L$gcm_dec_abort push rbp $L$SEH_prologue_aesni_gcm_decrypt_2: - mov rbp,rsp + mov rbp,rsp ; save stack pointer push rbx @@ -381,19 +381,19 @@ push r15 $L$SEH_prologue_aesni_gcm_decrypt_7: - lea rsp,[((-168))+rsp] + lea rsp,[((-168))+rsp] ; 8 extra bytes to align the stack $L$SEH_prologue_aesni_gcm_decrypt_8: $L$SEH_prologue_aesni_gcm_decrypt_9: - - - +; Load the last two parameters. These go into %rdi and %rsi, which are +; non-volatile on Windows, so stash them in the parameter stack area +; first. mov QWORD[16+rbp],rdi $L$SEH_prologue_aesni_gcm_decrypt_10: mov QWORD[24+rbp],rsi $L$SEH_prologue_aesni_gcm_decrypt_11: mov rdi,QWORD[48+rbp] mov rsi,QWORD[56+rbp] - +; Save non-volatile XMM registers. movaps XMMWORD[(-208)+rbp],xmm6 $L$SEH_prologue_aesni_gcm_decrypt_12: movaps XMMWORD[(-192)+rbp],xmm7 @@ -418,17 +418,17 @@ vzeroupper mov r12,QWORD[64+rbp] - vmovdqu xmm1,XMMWORD[rdi] + vmovdqu xmm1,XMMWORD[rdi] ; input counter value add rsp,-128 mov ebx,DWORD[12+rdi] lea r11,[$L$bswap_mask] - lea r14,[((-128))+r9] - mov r15,0xf80 - vmovdqu xmm8,XMMWORD[r12] - and rsp,-128 - vmovdqu xmm0,XMMWORD[r11] - lea r9,[128+r9] - lea rsi,[32+rsi] + lea r14,[((-128))+r9] ; borrow %r14 + mov r15,0xf80 ; borrow %r15 + vmovdqu xmm8,XMMWORD[r12] ; load Xi + and rsp,-128 ; ensure stack alignment + vmovdqu xmm0,XMMWORD[r11] ; borrow %xmm0 for .Lbswap_mask + lea r9,[128+r9] ; size optimization + lea rsi,[32+rsi] ; size optimization mov r10d,DWORD[((240-128))+r9] vpshufb xmm8,xmm8,xmm0 @@ -438,26 +438,26 @@ jc NEAR $L$dec_no_key_aliasing cmp r15,768 jnc NEAR $L$dec_no_key_aliasing - sub rsp,r15 + sub rsp,r15 ; avoid aliasing with key $L$dec_no_key_aliasing: - vmovdqu xmm7,XMMWORD[80+rcx] + vmovdqu xmm7,XMMWORD[80+rcx] ; I[5] mov r14,rcx vmovdqu xmm4,XMMWORD[64+rcx] - - - - - - +; |_aesni_ctr32_ghash_6x| requires |%r15| to point to 2*96 (0xc0) +; bytes before the end of the input. Note, in particular, that this is +; correct even if |%r8| is not an even multiple of 96 or 16. XXX: This +; seems to require that |%rcx| + |%r8| >= 2*96 (0xc0); i.e. |%rcx| must +; not be near the very beginning of the address space when |%r8| < 2*96 +; (0xc0). lea r15,[((-192))+r8*1+rcx] vmovdqu xmm5,XMMWORD[48+rcx] shr r8,4 xor rax,rax vmovdqu xmm6,XMMWORD[32+rcx] - vpshufb xmm7,xmm7,xmm0 + vpshufb xmm7,xmm7,xmm0 ; passed to _aesni_ctr32_ghash_6x vmovdqu xmm2,XMMWORD[16+rcx] vpshufb xmm4,xmm4,xmm0 vmovdqu xmm3,XMMWORD[rcx] @@ -474,15 +474,15 @@ call _aesni_ctr32_ghash_6x mov r12,QWORD[64+rbp] - vmovups XMMWORD[(-96)+rdx],xmm9 + vmovups XMMWORD[(-96)+rdx],xmm9 ; save output vmovups XMMWORD[(-80)+rdx],xmm10 vmovups XMMWORD[(-64)+rdx],xmm11 vmovups XMMWORD[(-48)+rdx],xmm12 vmovups XMMWORD[(-32)+rdx],xmm13 vmovups XMMWORD[(-16)+rdx],xmm14 - vpshufb xmm8,xmm8,XMMWORD[r11] - vmovdqu XMMWORD[r12],xmm8 + vpshufb xmm8,xmm8,XMMWORD[r11] ; .Lbswap_mask + vmovdqu XMMWORD[r12],xmm8 ; output Xi vzeroupper movaps xmm6,XMMWORD[((-208))+rbp] @@ -497,7 +497,7 @@ movaps xmm15,XMMWORD[((-64))+rbp] mov rdi,QWORD[16+rbp] mov rsi,QWORD[24+rbp] - lea rsp,[((-40))+rbp] + lea rsp,[((-40))+rbp] ; restore %rsp to fixed allocation pop r15 @@ -520,8 +520,8 @@ ALIGN 32 _aesni_ctr32_6x: - vmovdqu xmm4,XMMWORD[((0-128))+r9] - vmovdqu xmm2,XMMWORD[32+r11] + vmovdqu xmm4,XMMWORD[((0-128))+r9] ; borrow %xmm4 for %xmm15 + vmovdqu xmm2,XMMWORD[32+r11] ; borrow %xmm2, .Lone_msb lea r13,[((-1))+r10] vmovups xmm15,XMMWORD[((16-128))+r9] lea r12,[((32-128))+r9] @@ -554,7 +554,7 @@ dec r13d jnz NEAR $L$oop_ctr32 - vmovdqu xmm3,XMMWORD[r12] + vmovdqu xmm3,XMMWORD[r12] ; last round key vaesenc xmm9,xmm9,xmm15 vpxor xmm4,xmm3,XMMWORD[rcx] vaesenc xmm10,xmm10,xmm15 @@ -586,9 +586,9 @@ ret ALIGN 32 $L$handle_ctr32_2: - vpshufb xmm6,xmm1,xmm0 - vmovdqu xmm5,XMMWORD[48+r11] - vpaddd xmm10,xmm6,XMMWORD[64+r11] + vpshufb xmm6,xmm1,xmm0 ; byte-swap counter + vmovdqu xmm5,XMMWORD[48+r11] ; borrow %xmm5, .Ltwo_lsb + vpaddd xmm10,xmm6,XMMWORD[64+r11] ; .Lone_lsb vpaddd xmm11,xmm6,xmm5 vpaddd xmm12,xmm10,xmm5 vpshufb xmm10,xmm10,xmm0 @@ -598,12 +598,12 @@ vpaddd xmm14,xmm12,xmm5 vpshufb xmm12,xmm12,xmm0 vpxor xmm11,xmm11,xmm4 - vpaddd xmm1,xmm13,xmm5 + vpaddd xmm1,xmm13,xmm5 ; byte-swapped next counter value vpshufb xmm13,xmm13,xmm0 vpxor xmm12,xmm12,xmm4 vpshufb xmm14,xmm14,xmm0 vpxor xmm13,xmm13,xmm4 - vpshufb xmm1,xmm1,xmm0 + vpshufb xmm1,xmm1,xmm0 ; next counter value vpxor xmm14,xmm14,xmm4 jmp NEAR $L$oop_ctr32 @@ -622,16 +622,16 @@ %endif xor rax,rax - - - - cmp r8,0x60*3 +; We call |_aesni_ctr32_6x| twice, each call consuming 96 bytes of +; input. Then we call |_aesni_ctr32_ghash_6x|, which requires at +; least 96 more bytes of input. + cmp r8,0x60*3 ; minimal accepted length jb NEAR $L$gcm_enc_abort push rbp $L$SEH_prologue_aesni_gcm_encrypt_2: - mov rbp,rsp + mov rbp,rsp ; save stack pointer push rbx @@ -648,19 +648,19 @@ push r15 $L$SEH_prologue_aesni_gcm_encrypt_7: - lea rsp,[((-168))+rsp] + lea rsp,[((-168))+rsp] ; 8 extra bytes to align the stack $L$SEH_prologue_aesni_gcm_encrypt_8: $L$SEH_prologue_aesni_gcm_encrypt_9: - - - +; Load the last two parameters. These go into %rdi and %rsi, which are +; non-volatile on Windows, so stash them in the parameter stack area +; first. mov QWORD[16+rbp],rdi $L$SEH_prologue_aesni_gcm_encrypt_10: mov QWORD[24+rbp],rsi $L$SEH_prologue_aesni_gcm_encrypt_11: mov rdi,QWORD[48+rbp] mov rsi,QWORD[56+rbp] - +; Save non-volatile XMM registers. movaps XMMWORD[(-208)+rbp],xmm6 $L$SEH_prologue_aesni_gcm_encrypt_12: movaps XMMWORD[(-192)+rbp],xmm7 @@ -684,15 +684,15 @@ $L$SEH_endprologue_aesni_gcm_encrypt_22: vzeroupper - vmovdqu xmm1,XMMWORD[rdi] + vmovdqu xmm1,XMMWORD[rdi] ; input counter value add rsp,-128 mov ebx,DWORD[12+rdi] lea r11,[$L$bswap_mask] - lea r14,[((-128))+r9] - mov r15,0xf80 - lea r9,[128+r9] - vmovdqu xmm0,XMMWORD[r11] - and rsp,-128 + lea r14,[((-128))+r9] ; borrow %r14 + mov r15,0xf80 ; borrow %r15 + lea r9,[128+r9] ; size optimization + vmovdqu xmm0,XMMWORD[r11] ; borrow %xmm0 for .Lbswap_mask + and rsp,-128 ; ensure stack alignment mov r10d,DWORD[((240-128))+r9] and r14,r15 @@ -701,24 +701,24 @@ jc NEAR $L$enc_no_key_aliasing cmp r15,768 jnc NEAR $L$enc_no_key_aliasing - sub rsp,r15 + sub rsp,r15 ; avoid aliasing with key $L$enc_no_key_aliasing: mov r14,rdx - - - - - - - +; |_aesni_ctr32_ghash_6x| requires |%r15| to point to 2*96 (0xc0) +; bytes before the end of the input. Note, in particular, that this is +; correct even if |%r8| is not an even multiple of 96 or 16. Unlike in +; the decryption case, there's no caveat that |%rdx| must not be near +; the very beginning of the address space, because we know that +; |%r8| >= 3*96 from the check above, and so we know +; |%rdx| + |%r8| >= 2*96 (0xc0). lea r15,[((-192))+r8*1+rdx] shr r8,4 call _aesni_ctr32_6x - vpshufb xmm8,xmm9,xmm0 + vpshufb xmm8,xmm9,xmm0 ; save bswapped output on stack vpshufb xmm2,xmm10,xmm0 vmovdqu XMMWORD[112+rsp],xmm8 vpshufb xmm4,xmm11,xmm0 @@ -727,26 +727,26 @@ vmovdqu XMMWORD[80+rsp],xmm4 vpshufb xmm6,xmm13,xmm0 vmovdqu XMMWORD[64+rsp],xmm5 - vpshufb xmm7,xmm14,xmm0 + vpshufb xmm7,xmm14,xmm0 ; passed to _aesni_ctr32_ghash_6x vmovdqu XMMWORD[48+rsp],xmm6 call _aesni_ctr32_6x mov r12,QWORD[64+rbp] - lea rsi,[32+rsi] - vmovdqu xmm8,XMMWORD[r12] + lea rsi,[32+rsi] ; size optimization + vmovdqu xmm8,XMMWORD[r12] ; load Xi sub r8,12 mov rax,0x60*2 vpshufb xmm8,xmm8,xmm0 call _aesni_ctr32_ghash_6x - vmovdqu xmm7,XMMWORD[32+rsp] - vmovdqu xmm0,XMMWORD[r11] - vmovdqu xmm3,XMMWORD[((0-32))+rsi] + vmovdqu xmm7,XMMWORD[32+rsp] ; I[5] + vmovdqu xmm0,XMMWORD[r11] ; borrow %xmm0 for .Lbswap_mask + vmovdqu xmm3,XMMWORD[((0-32))+rsi] ; %xmm3^1 vpunpckhqdq xmm1,xmm7,xmm7 - vmovdqu xmm15,XMMWORD[((32-32))+rsi] - vmovups XMMWORD[(-96)+rdx],xmm9 - vpshufb xmm9,xmm9,xmm0 + vmovdqu xmm15,XMMWORD[((32-32))+rsi] ; borrow %xmm15 for + vmovups XMMWORD[(-96)+rdx],xmm9 ; save output + vpshufb xmm9,xmm9,xmm0 ; but keep bswapped copy vpxor xmm1,xmm1,xmm7 vmovups XMMWORD[(-80)+rdx],xmm10 vpshufb xmm10,xmm10,xmm0 @@ -758,18 +758,18 @@ vpshufb xmm13,xmm13,xmm0 vmovups XMMWORD[(-16)+rdx],xmm14 vpshufb xmm14,xmm14,xmm0 - vmovdqu XMMWORD[16+rsp],xmm9 - vmovdqu xmm6,XMMWORD[48+rsp] - vmovdqu xmm0,XMMWORD[((16-32))+rsi] + vmovdqu XMMWORD[16+rsp],xmm9 ; free %xmm9 + vmovdqu xmm6,XMMWORD[48+rsp] ; I[4] + vmovdqu xmm0,XMMWORD[((16-32))+rsi] ; borrow %xmm0 for %xmm3^2 vpunpckhqdq xmm2,xmm6,xmm6 vpclmulqdq xmm5,xmm7,xmm3,0x00 vpxor xmm2,xmm2,xmm6 vpclmulqdq xmm7,xmm7,xmm3,0x11 vpclmulqdq xmm1,xmm1,xmm15,0x00 - vmovdqu xmm9,XMMWORD[64+rsp] + vmovdqu xmm9,XMMWORD[64+rsp] ; I[3] vpclmulqdq xmm4,xmm6,xmm0,0x00 - vmovdqu xmm3,XMMWORD[((48-32))+rsi] + vmovdqu xmm3,XMMWORD[((48-32))+rsi] ; %xmm3^3 vpxor xmm4,xmm4,xmm5 vpunpckhqdq xmm5,xmm9,xmm9 vpclmulqdq xmm6,xmm6,xmm0,0x11 @@ -779,9 +779,9 @@ vmovdqu xmm15,XMMWORD[((80-32))+rsi] vpxor xmm2,xmm2,xmm1 - vmovdqu xmm1,XMMWORD[80+rsp] + vmovdqu xmm1,XMMWORD[80+rsp] ; I[2] vpclmulqdq xmm7,xmm9,xmm3,0x00 - vmovdqu xmm0,XMMWORD[((64-32))+rsi] + vmovdqu xmm0,XMMWORD[((64-32))+rsi] ; borrow %xmm0 for %xmm3^4 vpxor xmm7,xmm7,xmm4 vpunpckhqdq xmm4,xmm1,xmm1 vpclmulqdq xmm9,xmm9,xmm3,0x11 @@ -790,9 +790,9 @@ vpclmulqdq xmm5,xmm5,xmm15,0x00 vpxor xmm5,xmm5,xmm2 - vmovdqu xmm2,XMMWORD[96+rsp] + vmovdqu xmm2,XMMWORD[96+rsp] ; I[1] vpclmulqdq xmm6,xmm1,xmm0,0x00 - vmovdqu xmm3,XMMWORD[((96-32))+rsi] + vmovdqu xmm3,XMMWORD[((96-32))+rsi] ; %xmm3^5 vpxor xmm6,xmm6,xmm7 vpunpckhqdq xmm7,xmm2,xmm2 vpclmulqdq xmm1,xmm1,xmm0,0x11 @@ -802,9 +802,9 @@ vmovdqu xmm15,XMMWORD[((128-32))+rsi] vpxor xmm4,xmm4,xmm5 - vpxor xmm8,xmm8,XMMWORD[112+rsp] + vpxor xmm8,xmm8,XMMWORD[112+rsp] ; accumulate I[0] vpclmulqdq xmm5,xmm2,xmm3,0x00 - vmovdqu xmm0,XMMWORD[((112-32))+rsi] + vmovdqu xmm0,XMMWORD[((112-32))+rsi] ; borrow %xmm0 for %xmm3^6 vpunpckhqdq xmm9,xmm8,xmm8 vpxor xmm5,xmm5,xmm6 vpclmulqdq xmm2,xmm2,xmm3,0x11 @@ -814,7 +814,7 @@ vpxor xmm4,xmm7,xmm4 vpclmulqdq xmm6,xmm8,xmm0,0x00 - vmovdqu xmm3,XMMWORD[((0-32))+rsi] + vmovdqu xmm3,XMMWORD[((0-32))+rsi] ; %xmm3^1 vpunpckhqdq xmm1,xmm14,xmm14 vpclmulqdq xmm8,xmm8,xmm0,0x11 vpxor xmm1,xmm1,xmm14 @@ -824,8 +824,8 @@ vpxor xmm7,xmm8,xmm2 vpxor xmm6,xmm9,xmm4 - vmovdqu xmm0,XMMWORD[((16-32))+rsi] - vpxor xmm9,xmm7,xmm5 + vmovdqu xmm0,XMMWORD[((16-32))+rsi] ; borrow %xmm0 for %xmm3^2 + vpxor xmm9,xmm7,xmm5 ; aggregated Karatsuba post-processing vpclmulqdq xmm4,xmm14,xmm3,0x00 vpxor xmm6,xmm6,xmm9 vpunpckhqdq xmm2,xmm13,xmm13 @@ -838,25 +838,25 @@ vpxor xmm7,xmm7,xmm6 vpclmulqdq xmm5,xmm13,xmm0,0x00 - vmovdqu xmm3,XMMWORD[((48-32))+rsi] + vmovdqu xmm3,XMMWORD[((48-32))+rsi] ; %xmm3^3 vpxor xmm5,xmm5,xmm4 vpunpckhqdq xmm9,xmm12,xmm12 vpclmulqdq xmm13,xmm13,xmm0,0x11 vpxor xmm9,xmm9,xmm12 vpxor xmm13,xmm13,xmm14 - vpalignr xmm14,xmm8,xmm8,8 + vpalignr xmm14,xmm8,xmm8,8 ; 1st phase vpclmulqdq xmm2,xmm2,xmm15,0x10 vmovdqu xmm15,XMMWORD[((80-32))+rsi] vpxor xmm2,xmm2,xmm1 vpclmulqdq xmm4,xmm12,xmm3,0x00 - vmovdqu xmm0,XMMWORD[((64-32))+rsi] + vmovdqu xmm0,XMMWORD[((64-32))+rsi] ; borrow %xmm0 for %xmm3^4 vpxor xmm4,xmm4,xmm5 vpunpckhqdq xmm1,xmm11,xmm11 vpclmulqdq xmm12,xmm12,xmm3,0x11 vpxor xmm1,xmm1,xmm11 vpxor xmm12,xmm12,xmm13 - vxorps xmm7,xmm7,XMMWORD[16+rsp] + vxorps xmm7,xmm7,XMMWORD[16+rsp] ; accumulate %xmm9 vpclmulqdq xmm9,xmm9,xmm15,0x00 vpxor xmm9,xmm9,xmm2 @@ -864,12 +864,12 @@ vxorps xmm8,xmm8,xmm14 vpclmulqdq xmm5,xmm11,xmm0,0x00 - vmovdqu xmm3,XMMWORD[((96-32))+rsi] + vmovdqu xmm3,XMMWORD[((96-32))+rsi] ; %xmm3^5 vpxor xmm5,xmm5,xmm4 vpunpckhqdq xmm2,xmm10,xmm10 vpclmulqdq xmm11,xmm11,xmm0,0x11 vpxor xmm2,xmm2,xmm10 - vpalignr xmm14,xmm8,xmm8,8 + vpalignr xmm14,xmm8,xmm8,8 ; 2nd phase vpxor xmm11,xmm11,xmm12 vpclmulqdq xmm1,xmm1,xmm15,0x10 vmovdqu xmm15,XMMWORD[((128-32))+rsi] @@ -880,7 +880,7 @@ vxorps xmm8,xmm8,xmm14 vpclmulqdq xmm4,xmm10,xmm3,0x00 - vmovdqu xmm0,XMMWORD[((112-32))+rsi] + vmovdqu xmm0,XMMWORD[((112-32))+rsi] ; borrow %xmm0 for %xmm3^6 vpxor xmm4,xmm4,xmm5 vpunpckhqdq xmm9,xmm8,xmm8 vpclmulqdq xmm10,xmm10,xmm3,0x11 @@ -896,25 +896,25 @@ vpxor xmm7,xmm7,xmm10 vpxor xmm6,xmm6,xmm2 - vpxor xmm4,xmm7,xmm5 + vpxor xmm4,xmm7,xmm5 ; aggregated Karatsuba post-processing vpxor xmm6,xmm6,xmm4 vpslldq xmm1,xmm6,8 - vmovdqu xmm3,XMMWORD[16+r11] + vmovdqu xmm3,XMMWORD[16+r11] ; .Lpoly vpsrldq xmm6,xmm6,8 vpxor xmm8,xmm5,xmm1 vpxor xmm7,xmm7,xmm6 - vpalignr xmm2,xmm8,xmm8,8 + vpalignr xmm2,xmm8,xmm8,8 ; 1st phase vpclmulqdq xmm8,xmm8,xmm3,0x10 vpxor xmm8,xmm8,xmm2 - vpalignr xmm2,xmm8,xmm8,8 + vpalignr xmm2,xmm8,xmm8,8 ; 2nd phase vpclmulqdq xmm8,xmm8,xmm3,0x10 vpxor xmm2,xmm2,xmm7 vpxor xmm8,xmm8,xmm2 mov r12,QWORD[64+rbp] - vpshufb xmm8,xmm8,XMMWORD[r11] - vmovdqu XMMWORD[r12],xmm8 + vpshufb xmm8,xmm8,XMMWORD[r11] ; .Lbswap_mask + vmovdqu XMMWORD[r12],xmm8 ; output Xi vzeroupper movaps xmm6,XMMWORD[((-208))+rbp] @@ -929,7 +929,7 @@ movaps xmm15,XMMWORD[((-64))+rbp] mov rdi,QWORD[16+rbp] mov rsi,QWORD[24+rbp] - lea rsp,[((-40))+rbp] + lea rsp,[((-40))+rbp] ; restore %rsp to fixed allocation pop r15 @@ -983,7 +983,7 @@ section .xdata rdata align=8 ALIGN 4 $L$SEH_info_aesni_gcm_decrypt_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aesni_gcm_decrypt_22-$L$SEH_begin_aesni_gcm_decrypt_1 DB 33 DB 213 @@ -1043,7 +1043,7 @@ DW 0 $L$SEH_info_aesni_gcm_encrypt_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aesni_gcm_encrypt_22-$L$SEH_begin_aesni_gcm_encrypt_1 DB 33 DB 213
diff --git a/gen/bcm/aesni-x86_64-apple.S b/gen/bcm/aesni-x86_64-apple.S index 256a323..5ae78fb 100644 --- a/gen/bcm/aesni-x86_64-apple.S +++ b/gen/bcm/aesni-x86_64-apple.S
@@ -16,8 +16,8 @@ movb $1,_BORINGSSL_function_hit+1(%rip) #endif - movups (%rdi),%xmm2 - movl 240(%rdx),%eax + movups (%rdi),%xmm2 // load input + movl 240(%rdx),%eax // key->rounds movups (%rdx),%xmm0 movups 16(%rdx),%xmm1 leaq 32(%rdx),%rdx @@ -27,11 +27,11 @@ decl %eax movups (%rdx),%xmm1 leaq 16(%rdx),%rdx - jnz L$oop_enc1_1 + jnz L$oop_enc1_1 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // output pxor %xmm2,%xmm2 ret @@ -45,8 +45,8 @@ _aes_hw_decrypt: _CET_ENDBR - movups (%rdi),%xmm2 - movl 240(%rdx),%eax + movups (%rdi),%xmm2 // load input + movl 240(%rdx),%eax // key->rounds movups (%rdx),%xmm0 movups 16(%rdx),%xmm1 leaq 32(%rdx),%rdx @@ -56,11 +56,11 @@ decl %eax movups (%rdx),%xmm1 leaq 16(%rdx),%rdx - jnz L$oop_dec1_2 + jnz L$oop_dec1_2 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // output pxor %xmm2,%xmm2 ret @@ -77,7 +77,7 @@ xorps %xmm0,%xmm3 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax L$enc_loop2: @@ -109,7 +109,7 @@ xorps %xmm0,%xmm3 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax L$dec_loop2: @@ -142,7 +142,7 @@ xorps %xmm0,%xmm4 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax L$enc_loop3: @@ -179,7 +179,7 @@ xorps %xmm0,%xmm4 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax L$dec_loop3: @@ -217,7 +217,7 @@ xorps %xmm0,%xmm5 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax .byte 0x0f,0x1f,0x00 addq $16,%rax @@ -260,7 +260,7 @@ xorps %xmm0,%xmm5 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax .byte 0x0f,0x1f,0x00 addq $16,%rax @@ -302,7 +302,7 @@ pxor %xmm0,%xmm4 aesenc %xmm1,%xmm2 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesenc %xmm1,%xmm3 pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 @@ -359,7 +359,7 @@ pxor %xmm0,%xmm4 aesdec %xmm1,%xmm2 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesdec %xmm1,%xmm3 pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 @@ -417,7 +417,7 @@ pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesenc %xmm1,%xmm2 pxor %xmm0,%xmm7 pxor %xmm0,%xmm8 @@ -484,7 +484,7 @@ pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesdec %xmm1,%xmm2 pxor %xmm0,%xmm7 pxor %xmm0,%xmm8 @@ -545,20 +545,20 @@ _aes_hw_ecb_encrypt: _CET_ENDBR - andq $-16,%rdx - jz L$ecb_ret + andq $-16,%rdx // if (%rdx<16) + jz L$ecb_ret // return - movl 240(%rcx),%eax + movl 240(%rcx),%eax // key->rounds movups (%rcx),%xmm0 - movq %rcx,%r11 - movl %eax,%r10d - testl %r8d,%r8d + movq %rcx,%r11 // backup %rcx + movl %eax,%r10d // backup %eax + testl %r8d,%r8d // 5th argument jz L$ecb_decrypt +// --------------------------- ECB ENCRYPT ------------------------------ + cmpq $0x80,%rdx // if (%rdx<8*16) + jb L$ecb_enc_tail // short input - cmpq $0x80,%rdx - jb L$ecb_enc_tail - - movdqu (%rdi),%xmm2 + movdqu (%rdi),%xmm2 // load 8 input blocks movdqu 16(%rdi),%xmm3 movdqu 32(%rdi),%xmm4 movdqu 48(%rdi),%xmm5 @@ -566,15 +566,15 @@ movdqu 80(%rdi),%xmm7 movdqu 96(%rdi),%xmm8 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi - subq $0x80,%rdx + leaq 128(%rdi),%rdi // %rdi+=8*16 + subq $0x80,%rdx // %rdx-=8*16 (can be zero) jmp L$ecb_enc_loop8_enter .p2align 4 L$ecb_enc_loop8: - movups %xmm2,(%rsi) - movq %r11,%rcx - movdqu (%rdi),%xmm2 - movl %r10d,%eax + movups %xmm2,(%rsi) // store 8 output blocks + movq %r11,%rcx // restore %rcx + movdqu (%rdi),%xmm2 // load 8 input blocks + movl %r10d,%eax // restore %eax movups %xmm3,16(%rsi) movdqu 16(%rdi),%xmm3 movups %xmm4,32(%rsi) @@ -588,31 +588,31 @@ movups %xmm8,96(%rsi) movdqu 96(%rdi),%xmm8 movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // %rsi+=8*16 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // %rdi+=8*16 L$ecb_enc_loop8_enter: call _aesni_encrypt8 subq $0x80,%rdx - jnc L$ecb_enc_loop8 + jnc L$ecb_enc_loop8 // loop if %rdx-=8*16 didn't borrow - movups %xmm2,(%rsi) - movq %r11,%rcx + movups %xmm2,(%rsi) // store 8 output blocks + movq %r11,%rcx // restore %rcx movups %xmm3,16(%rsi) - movl %r10d,%eax + movl %r10d,%eax // restore %eax movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) movups %xmm6,64(%rsi) movups %xmm7,80(%rsi) movups %xmm8,96(%rsi) movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi - addq $0x80,%rdx - jz L$ecb_ret + leaq 128(%rsi),%rsi // %rsi+=8*16 + addq $0x80,%rdx // restore real remaining %rdx + jz L$ecb_ret // done if (%rdx==0) -L$ecb_enc_tail: +L$ecb_enc_tail: // %rdx is less than 8*16 movups (%rdi),%xmm2 cmpq $0x20,%rdx jb L$ecb_enc_one @@ -631,7 +631,7 @@ movdqu 96(%rdi),%xmm8 xorps %xmm9,%xmm9 call _aesni_encrypt8 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 7 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) @@ -650,27 +650,27 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz L$oop_enc1_3 + jnz L$oop_enc1_3 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store one output block jmp L$ecb_ret .p2align 4 L$ecb_enc_two: call _aesni_encrypt2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 2 output blocks movups %xmm3,16(%rsi) jmp L$ecb_ret .p2align 4 L$ecb_enc_three: call _aesni_encrypt3 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 3 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) jmp L$ecb_ret .p2align 4 L$ecb_enc_four: call _aesni_encrypt4 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 4 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) @@ -679,7 +679,7 @@ L$ecb_enc_five: xorps %xmm7,%xmm7 call _aesni_encrypt6 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 5 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) @@ -688,20 +688,20 @@ .p2align 4 L$ecb_enc_six: call _aesni_encrypt6 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 6 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) movups %xmm6,64(%rsi) movups %xmm7,80(%rsi) jmp L$ecb_ret - +// --------------------------- ECB DECRYPT ------------------------------# .p2align 4 L$ecb_decrypt: - cmpq $0x80,%rdx - jb L$ecb_dec_tail + cmpq $0x80,%rdx // if (%rdx<8*16) + jb L$ecb_dec_tail // short input - movdqu (%rdi),%xmm2 + movdqu (%rdi),%xmm2 // load 8 input blocks movdqu 16(%rdi),%xmm3 movdqu 32(%rdi),%xmm4 movdqu 48(%rdi),%xmm5 @@ -709,15 +709,15 @@ movdqu 80(%rdi),%xmm7 movdqu 96(%rdi),%xmm8 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi - subq $0x80,%rdx + leaq 128(%rdi),%rdi // %rdi+=8*16 + subq $0x80,%rdx // %rdx-=8*16 (can be zero) jmp L$ecb_dec_loop8_enter .p2align 4 L$ecb_dec_loop8: - movups %xmm2,(%rsi) - movq %r11,%rcx - movdqu (%rdi),%xmm2 - movl %r10d,%eax + movups %xmm2,(%rsi) // store 8 output blocks + movq %r11,%rcx // restore %rcx + movdqu (%rdi),%xmm2 // load 8 input blocks + movl %r10d,%eax // restore %eax movups %xmm3,16(%rsi) movdqu 16(%rdi),%xmm3 movups %xmm4,32(%rsi) @@ -731,23 +731,23 @@ movups %xmm8,96(%rsi) movdqu 96(%rdi),%xmm8 movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // %rsi+=8*16 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // %rdi+=8*16 L$ecb_dec_loop8_enter: call _aesni_decrypt8 movups (%r11),%xmm0 subq $0x80,%rdx - jnc L$ecb_dec_loop8 + jnc L$ecb_dec_loop8 // loop if %rdx-=8*16 didn't borrow - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 - movq %r11,%rcx + movups %xmm2,(%rsi) // store 8 output blocks + pxor %xmm2,%xmm2 // clear register bank + movq %r11,%rcx // restore %rcx movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 - movl %r10d,%eax + movl %r10d,%eax // restore %eax movups %xmm4,32(%rsi) pxor %xmm4,%xmm4 movups %xmm5,48(%rsi) @@ -760,9 +760,9 @@ pxor %xmm8,%xmm8 movups %xmm9,112(%rsi) pxor %xmm9,%xmm9 - leaq 128(%rsi),%rsi - addq $0x80,%rdx - jz L$ecb_ret + leaq 128(%rsi),%rsi // %rsi+=8*16 + addq $0x80,%rdx // restore real remaining %rdx + jz L$ecb_ret // done if (%rdx==0) L$ecb_dec_tail: movups (%rdi),%xmm2 @@ -784,8 +784,8 @@ movups (%rcx),%xmm0 xorps %xmm9,%xmm9 call _aesni_decrypt8 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 7 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -811,24 +811,24 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz L$oop_dec1_4 + jnz L$oop_dec1_4 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store one output block + pxor %xmm2,%xmm2 // clear register bank jmp L$ecb_ret .p2align 4 L$ecb_dec_two: call _aesni_decrypt2 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 2 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 jmp L$ecb_ret .p2align 4 L$ecb_dec_three: call _aesni_decrypt3 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 3 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -837,8 +837,8 @@ .p2align 4 L$ecb_dec_four: call _aesni_decrypt4 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 4 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -850,8 +850,8 @@ L$ecb_dec_five: xorps %xmm7,%xmm7 call _aesni_decrypt6 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 5 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -865,8 +865,8 @@ .p2align 4 L$ecb_dec_six: call _aesni_decrypt6 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 6 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -879,7 +879,7 @@ pxor %xmm7,%xmm7 L$ecb_ret: - xorps %xmm0,%xmm0 + xorps %xmm0,%xmm0 // %xmm0 pxor %xmm1,%xmm1 ret @@ -898,11 +898,11 @@ cmpq $1,%rdx jne L$ctr32_bulk - - +// handle single block without allocating stack frame, +// useful when handling edges movups (%r8),%xmm2 movups (%rdi),%xmm3 - movl 240(%rcx),%edx + movl 240(%rcx),%edx // key->rounds movups (%rcx),%xmm0 movups 16(%rcx),%xmm1 leaq 32(%rcx),%rcx @@ -912,9 +912,9 @@ decl %edx movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz L$oop_enc1_5 + jnz L$oop_enc1_5 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 xorps %xmm3,%xmm2 pxor %xmm3,%xmm3 @@ -924,22 +924,22 @@ .p2align 4 L$ctr32_bulk: - leaq (%rsp),%r11 + leaq (%rsp),%r11 // use %r11 as frame pointer pushq %rbp subq $128,%rsp - andq $-16,%rsp + andq $-16,%rsp // Linux kernel stack can be incorrectly seeded - - +// 8 16-byte words on top of stack are counter values +// xor-ed with zero-round key movdqu (%r8),%xmm2 movdqu (%rcx),%xmm0 - movl 12(%r8),%r8d + movl 12(%r8),%r8d // counter LSB pxor %xmm0,%xmm2 - movl 12(%rcx),%ebp - movdqa %xmm2,0(%rsp) + movl 12(%rcx),%ebp // 0-round key LSB + movdqa %xmm2,0(%rsp) // populate counter block bswapl %r8d movdqa %xmm2,%xmm3 movdqa %xmm2,%xmm4 @@ -947,7 +947,7 @@ movdqa %xmm2,64(%rsp) movdqa %xmm2,80(%rsp) movdqa %xmm2,96(%rsp) - movq %rdx,%r10 + movq %rdx,%r10 // about to borrow %rdx movdqa %xmm2,112(%rsp) leaq 1(%r8),%rax @@ -961,7 +961,7 @@ movdqa %xmm3,16(%rsp) pinsrd $3,%edx,%xmm4 bswapl %eax - movq %r10,%rdx + movq %r10,%rdx // restore %rdx leaq 4(%r8),%r10 movdqa %xmm4,32(%rsp) xorl %ebp,%eax @@ -973,7 +973,7 @@ movl %r10d,64+12(%rsp) bswapl %r9d leaq 6(%r8),%r10 - movl 240(%rcx),%eax + movl 240(%rcx),%eax // key->rounds xorl %ebp,%r9d bswapl %r10d movl %r9d,80+12(%rsp) @@ -989,16 +989,16 @@ movdqa 64(%rsp),%xmm6 movdqa 80(%rsp),%xmm7 - cmpq $8,%rdx - jb L$ctr32_tail + cmpq $8,%rdx // %rdx is in blocks + jb L$ctr32_tail // short input if (%rdx<8) - leaq 128(%rcx),%rcx - subq $8,%rdx + leaq 128(%rcx),%rcx // size optimization + subq $8,%rdx // %rdx is biased by -8 jmp L$ctr32_loop8 .p2align 5 L$ctr32_loop8: - addl $8,%r8d + addl $8,%r8d // next counter value movdqa 96(%rsp),%xmm8 aesenc %xmm1,%xmm2 movl %r8d,%r9d @@ -1010,7 +1010,7 @@ xorl %ebp,%r9d nop aesenc %xmm1,%xmm5 - movl %r9d,0+12(%rsp) + movl %r9d,0+12(%rsp) // store next counter value leaq 1(%r8),%r9 aesenc %xmm1,%xmm6 aesenc %xmm1,%xmm7 @@ -1106,7 +1106,7 @@ aesenc %xmm0,%xmm3 aesenc %xmm0,%xmm4 xorl %ebp,%r9d - movdqu 0(%rdi),%xmm10 + movdqu 0(%rdi),%xmm10 // start loading input aesenc %xmm0,%xmm5 movl %r9d,112+12(%rsp) cmpl $11,%eax @@ -1163,7 +1163,7 @@ .p2align 4 L$ctr32_enc_done: movdqu 16(%rdi),%xmm11 - pxor %xmm0,%xmm10 + pxor %xmm0,%xmm10 // input^=round[last] movdqu 32(%rdi),%xmm12 pxor %xmm0,%xmm11 movdqu 48(%rdi),%xmm13 @@ -1172,8 +1172,8 @@ pxor %xmm0,%xmm13 movdqu 80(%rdi),%xmm15 pxor %xmm0,%xmm14 - prefetcht0 448(%rdi) - prefetcht0 512(%rdi) + prefetcht0 448(%rdi) // We process 128 bytes (8*16), so to prefetch 1 iteration + prefetcht0 512(%rdi) // We need to prefetch 2 64 byte lines pxor %xmm0,%xmm15 aesenc %xmm1,%xmm2 aesenc %xmm1,%xmm3 @@ -1183,15 +1183,15 @@ aesenc %xmm1,%xmm7 aesenc %xmm1,%xmm8 aesenc %xmm1,%xmm9 - movdqu 96(%rdi),%xmm1 - leaq 128(%rdi),%rdi + movdqu 96(%rdi),%xmm1 // borrow %xmm1 for inp[6] + leaq 128(%rdi),%rdi // %rdi+=8*16 - aesenclast %xmm10,%xmm2 - pxor %xmm0,%xmm1 + aesenclast %xmm10,%xmm2 // is inp[N]^round[last] + pxor %xmm0,%xmm1 // borrowed movdqu 112-128(%rdi),%xmm10 aesenclast %xmm11,%xmm3 pxor %xmm0,%xmm10 - movdqa 0(%rsp),%xmm11 + movdqa 0(%rsp),%xmm11 // load next counter block aesenclast %xmm12,%xmm4 aesenclast %xmm13,%xmm5 movdqa 16(%rsp),%xmm12 @@ -1202,10 +1202,10 @@ movdqa 64(%rsp),%xmm15 aesenclast %xmm1,%xmm8 movdqa 80(%rsp),%xmm0 - movups 16-128(%rcx),%xmm1 + movups 16-128(%rcx),%xmm1 // real 1st-round key aesenclast %xmm10,%xmm9 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 8 output blocks movdqa %xmm11,%xmm2 movups %xmm3,16(%rsi) movdqa %xmm12,%xmm3 @@ -1219,24 +1219,24 @@ movdqa %xmm0,%xmm7 movups %xmm8,96(%rsi) movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // %rsi+=8*16 subq $8,%rdx - jnc L$ctr32_loop8 + jnc L$ctr32_loop8 // loop if %rdx-=8 didn't borrow - addq $8,%rdx - jz L$ctr32_done + addq $8,%rdx // restore real remaining %rdx + jz L$ctr32_done // done if (%rdx==0) leaq -128(%rcx),%rcx L$ctr32_tail: - - +// note that at this point %xmm2..5 are populated with +// counter values xor-ed with 0-round key leaq 16(%rcx),%rcx cmpq $4,%rdx jb L$ctr32_loop3 je L$ctr32_loop4 - +// if (%rdx>4) compute 7 E(counter) shll $4,%eax movdqa 96(%rsp),%xmm8 pxor %xmm9,%xmm9 @@ -1244,14 +1244,14 @@ movups 16(%rcx),%xmm0 aesenc %xmm1,%xmm2 aesenc %xmm1,%xmm3 - leaq 32-16(%rcx,%rax,1),%rcx + leaq 32-16(%rcx,%rax,1),%rcx // prepare for .Lenc_loop8_enter negq %rax aesenc %xmm1,%xmm4 - addq $16,%rax + addq $16,%rax // prepare for .Lenc_loop8_enter movups (%rdi),%xmm10 aesenc %xmm1,%xmm5 aesenc %xmm1,%xmm6 - movups 16(%rdi),%xmm11 + movups 16(%rdi),%xmm11 // pre-load input movups 32(%rdi),%xmm12 aesenc %xmm1,%xmm7 aesenc %xmm1,%xmm8 @@ -1262,7 +1262,7 @@ pxor %xmm10,%xmm2 movdqu 64(%rdi),%xmm10 pxor %xmm11,%xmm3 - movdqu %xmm2,(%rsi) + movdqu %xmm2,(%rsi) // store output pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) pxor %xmm13,%xmm5 @@ -1271,17 +1271,17 @@ movdqu %xmm5,48(%rsi) movdqu %xmm6,64(%rsi) cmpq $6,%rdx - jb L$ctr32_done + jb L$ctr32_done // %rdx was 5, stop store movups 80(%rdi),%xmm11 xorps %xmm11,%xmm7 movups %xmm7,80(%rsi) - je L$ctr32_done + je L$ctr32_done // %rdx was 6, stop store movups 96(%rdi),%xmm12 xorps %xmm12,%xmm8 movups %xmm8,96(%rsi) - jmp L$ctr32_done + jmp L$ctr32_done // %rdx was 7, stop store .p2align 5 L$ctr32_loop4: @@ -1295,7 +1295,7 @@ jnz L$ctr32_loop4 aesenclast %xmm1,%xmm2 aesenclast %xmm1,%xmm3 - movups (%rdi),%xmm10 + movups (%rdi),%xmm10 // load input movups 16(%rdi),%xmm11 aesenclast %xmm1,%xmm4 aesenclast %xmm1,%xmm5 @@ -1303,14 +1303,14 @@ movups 48(%rdi),%xmm13 xorps %xmm10,%xmm2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output xorps %xmm11,%xmm3 movups %xmm3,16(%rsi) pxor %xmm12,%xmm4 movdqu %xmm4,32(%rsi) pxor %xmm13,%xmm5 movdqu %xmm5,48(%rsi) - jmp L$ctr32_done + jmp L$ctr32_done // %rdx was 4, stop store .p2align 5 L$ctr32_loop3: @@ -1325,23 +1325,23 @@ aesenclast %xmm1,%xmm3 aesenclast %xmm1,%xmm4 - movups (%rdi),%xmm10 + movups (%rdi),%xmm10 // load input xorps %xmm10,%xmm2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output cmpq $2,%rdx - jb L$ctr32_done + jb L$ctr32_done // %rdx was 1, stop store movups 16(%rdi),%xmm11 xorps %xmm11,%xmm3 movups %xmm3,16(%rsi) - je L$ctr32_done + je L$ctr32_done // %rdx was 2, stop store movups 32(%rdi),%xmm12 xorps %xmm12,%xmm4 - movups %xmm4,32(%rsi) + movups %xmm4,32(%rsi) // %rdx was 3, stop store L$ctr32_done: - xorps %xmm0,%xmm0 + xorps %xmm0,%xmm0 // clear register bank xorl %ebp,%ebp pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -1350,7 +1350,7 @@ pxor %xmm5,%xmm5 pxor %xmm6,%xmm6 pxor %xmm7,%xmm7 - movaps %xmm0,0(%rsp) + movaps %xmm0,0(%rsp) // clear stack pxor %xmm8,%xmm8 movaps %xmm0,16(%rsp) pxor %xmm9,%xmm9 @@ -1382,15 +1382,15 @@ _aes_hw_cbc_encrypt: _CET_ENDBR - testq %rdx,%rdx + testq %rdx,%rdx // check length jz L$cbc_ret - movl 240(%rcx),%r10d - movq %rcx,%r11 - testl %r9d,%r9d + movl 240(%rcx),%r10d // key->rounds + movq %rcx,%r11 // backup %rcx + testl %r9d,%r9d // 6th argument jz L$cbc_decrypt - - movups (%r8),%xmm2 +// --------------------------- CBC ENCRYPT ------------------------------ + movups (%r8),%xmm2 // load iv as initial state movl %r10d,%eax cmpq $16,%rdx jb L$cbc_enc_tail @@ -1398,9 +1398,9 @@ jmp L$cbc_enc_loop .p2align 4 L$cbc_enc_loop: - movups (%rdi),%xmm3 + movups (%rdi),%xmm3 // load input leaq 16(%rdi),%rdi - +// xorps %xmm3,%xmm2 movups (%rcx),%xmm0 movups 16(%rcx),%xmm1 xorps %xmm0,%xmm3 @@ -1411,17 +1411,17 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz L$oop_enc1_6 + jnz L$oop_enc1_6 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - movl %r10d,%eax - movq %r11,%rcx - movups %xmm2,0(%rsi) + movl %r10d,%eax // restore %eax + movq %r11,%rcx // restore %rcx + movups %xmm2,0(%rsi) // store output leaq 16(%rsi),%rsi subq $16,%rdx jnc L$cbc_enc_loop addq $16,%rdx jnz L$cbc_enc_tail - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 movups %xmm2,(%r8) pxor %xmm2,%xmm2 @@ -1429,30 +1429,30 @@ jmp L$cbc_ret L$cbc_enc_tail: - movq %rdx,%rcx - xchgq %rdi,%rsi -.long 0x9066A4F3 - movl $16,%ecx + movq %rdx,%rcx // zaps %rcx + xchgq %rdi,%rsi // %rdi is %rsi and %rsi is %rdi now +.long 0x9066A4F3 // rep movsb + movl $16,%ecx // zero tail subq %rdx,%rcx xorl %eax,%eax -.long 0x9066AAF3 - leaq -16(%rdi),%rdi - movl %r10d,%eax - movq %rdi,%rsi - movq %r11,%rcx - xorq %rdx,%rdx - jmp L$cbc_enc_loop - +.long 0x9066AAF3 // rep stosb + leaq -16(%rdi),%rdi // rewind %rsi by 1 block + movl %r10d,%eax // restore %eax + movq %rdi,%rsi // %rdi and %rsi are the same + movq %r11,%rcx // restore %rcx + xorq %rdx,%rdx // len=16 + jmp L$cbc_enc_loop // one more spin +// --------------------------- CBC DECRYPT ------------------------------# .p2align 4 L$cbc_decrypt: cmpq $16,%rdx jne L$cbc_decrypt_bulk - - - movdqu (%rdi),%xmm2 - movdqu (%r8),%xmm3 - movdqa %xmm2,%xmm4 +// handle single block without allocating stack frame, +// useful in ciphertext stealing mode + movdqu (%rdi),%xmm2 // load input + movdqu (%r8),%xmm3 // load iv + movdqa %xmm2,%xmm4 // future iv movups (%rcx),%xmm0 movups 16(%rcx),%xmm1 leaq 32(%rcx),%rcx @@ -1462,32 +1462,32 @@ decl %r10d movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz L$oop_dec1_7 + jnz L$oop_dec1_7 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 - movdqu %xmm4,(%r8) - xorps %xmm3,%xmm2 + movdqu %xmm4,(%r8) // store iv + xorps %xmm3,%xmm2 // ^=iv pxor %xmm3,%xmm3 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output pxor %xmm2,%xmm2 jmp L$cbc_ret .p2align 4 L$cbc_decrypt_bulk: - leaq (%rsp),%r11 + leaq (%rsp),%r11 // frame pointer pushq %rbp subq $16,%rsp - andq $-16,%rsp - movq %rcx,%rbp + andq $-16,%rsp // Linux kernel stack can be incorrectly seeded + movq %rcx,%rbp // [re-]backup %rcx [after reassignment] movups (%r8),%xmm10 movl %r10d,%eax cmpq $0x50,%rdx jbe L$cbc_dec_tail movups (%rcx),%xmm0 - movdqu 0(%rdi),%xmm2 + movdqu 0(%rdi),%xmm2 // load input movdqu 16(%rdi),%xmm3 movdqa %xmm2,%xmm11 movdqu 32(%rdi),%xmm4 @@ -1501,8 +1501,8 @@ cmpq $0x70,%rdx jbe L$cbc_dec_six_or_seven - subq $0x70,%rdx - leaq 112(%rcx),%rcx + subq $0x70,%rdx // %rdx is biased by -7*16 + leaq 112(%rcx),%rcx // size optimization jmp L$cbc_dec_loop8_enter .p2align 4 L$cbc_dec_loop8: @@ -1516,7 +1516,7 @@ movups 16-112(%rcx),%xmm1 pxor %xmm0,%xmm4 movq $-1,%rbp - cmpq $0x70,%rdx + cmpq $0x70,%rdx // is there at least 0x60 bytes ahead? pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 pxor %xmm0,%xmm7 @@ -1665,11 +1665,11 @@ movdqu 80(%rdi),%xmm1 aesdeclast %xmm10,%xmm2 - movdqu 96(%rdi),%xmm10 + movdqu 96(%rdi),%xmm10 // borrow %xmm10 pxor %xmm0,%xmm1 aesdeclast %xmm11,%xmm3 pxor %xmm0,%xmm10 - movdqu 112(%rdi),%xmm0 + movdqu 112(%rdi),%xmm0 // next IV aesdeclast %xmm12,%xmm4 leaq 128(%rdi),%rdi movdqu 0(%rbp),%xmm11 @@ -1682,11 +1682,11 @@ movdqu 48(%rbp),%xmm14 movdqu 64(%rbp),%xmm15 aesdeclast %xmm10,%xmm9 - movdqa %xmm0,%xmm10 + movdqa %xmm0,%xmm10 // return %xmm10 movdqu 80(%rbp),%xmm1 movups -112(%rcx),%xmm0 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output movdqa %xmm11,%xmm2 movups %xmm3,16(%rsi) movdqa %xmm12,%xmm3 @@ -1720,13 +1720,13 @@ movaps %xmm7,%xmm8 call _aesni_decrypt6 - pxor %xmm10,%xmm2 + pxor %xmm10,%xmm2 // ^= IV movaps %xmm8,%xmm10 pxor %xmm11,%xmm3 movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1747,13 +1747,13 @@ xorps %xmm9,%xmm9 call _aesni_decrypt8 movups 80(%rdi),%xmm9 - pxor %xmm10,%xmm2 + pxor %xmm10,%xmm2 // ^= IV movups 96(%rdi),%xmm10 pxor %xmm11,%xmm3 movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1775,24 +1775,24 @@ L$cbc_dec_tail: movups (%rdi),%xmm2 subq $0x10,%rdx - jbe L$cbc_dec_one + jbe L$cbc_dec_one // %rdx is 1*16 or less movups 16(%rdi),%xmm3 movaps %xmm2,%xmm11 subq $0x10,%rdx - jbe L$cbc_dec_two + jbe L$cbc_dec_two // %rdx is 2*16 or less movups 32(%rdi),%xmm4 movaps %xmm3,%xmm12 subq $0x10,%rdx - jbe L$cbc_dec_three + jbe L$cbc_dec_three // %rdx is 3*16 or less movups 48(%rdi),%xmm5 movaps %xmm4,%xmm13 subq $0x10,%rdx - jbe L$cbc_dec_four + jbe L$cbc_dec_four // %rdx is 4*16 or less - movups 64(%rdi),%xmm6 + movups 64(%rdi),%xmm6 // %rdx is 5*16 or less movaps %xmm5,%xmm14 movaps %xmm6,%xmm15 xorps %xmm7,%xmm7 @@ -1803,7 +1803,7 @@ movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1829,7 +1829,7 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz L$oop_dec1_8 + jnz L$oop_dec1_8 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 xorps %xmm10,%xmm2 movaps %xmm11,%xmm10 @@ -1843,7 +1843,7 @@ pxor %xmm11,%xmm3 movdqu %xmm2,(%rsi) movdqa %xmm3,%xmm2 - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank leaq 16(%rsi),%rsi jmp L$cbc_dec_tail_collected .p2align 4 @@ -1856,7 +1856,7 @@ movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank movdqa %xmm4,%xmm2 pxor %xmm4,%xmm4 leaq 32(%rsi),%rsi @@ -1871,7 +1871,7 @@ movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1882,10 +1882,10 @@ .p2align 4 L$cbc_dec_clear_tail_collected: - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm4,%xmm4 pxor %xmm5,%xmm5 - pxor %xmm6,%xmm6 + pxor %xmm6,%xmm6 // %xmm6..9 pxor %xmm7,%xmm7 pxor %xmm8,%xmm8 pxor %xmm9,%xmm9 @@ -1904,11 +1904,11 @@ movq %rsi,%rdi subq %rdx,%rcx leaq (%rsp),%rsi -.long 0x9066A4F3 +.long 0x9066A4F3 // rep movsb movdqa %xmm2,(%rsp) L$cbc_dec_ret: - xorps %xmm0,%xmm0 + xorps %xmm0,%xmm0 // %xmm0 pxor %xmm1,%xmm1 movq -8(%r11),%rbp @@ -1930,9 +1930,9 @@ movl 240(%rdi),%esi shll $4,%esi - leaq 16(%rdi,%rsi,1),%rdx + leaq 16(%rdi,%rsi,1),%rdx // points at the end of key schedule - movups (%rdi),%xmm0 + movups (%rdi),%xmm0 // just swap movups (%rdx),%xmm1 movups %xmm0,(%rdx) movups %xmm1,(%rdi) @@ -1940,7 +1940,7 @@ leaq -16(%rdx),%rdx L$dec_key_inverse: - movups (%rdi),%xmm0 + movups (%rdi),%xmm0 // swap and inverse movups (%rdx),%xmm1 aesimc %xmm0,%xmm0 aesimc %xmm1,%xmm1 @@ -1951,7 +1951,7 @@ cmpq %rdi,%rdx ja L$dec_key_inverse - movups (%rdi),%xmm0 + movups (%rdi),%xmm0 // inverse middle aesimc %xmm0,%xmm0 pxor %xmm1,%xmm1 movups %xmm0,(%rdx) @@ -1975,9 +1975,9 @@ - movups (%rdi),%xmm0 - xorps %xmm4,%xmm4 - leaq 16(%rdx),%rax + movups (%rdi),%xmm0 // pull first 128 bits of *userKey + xorps %xmm4,%xmm4 // low dword of xmm4 is assumed 0 + leaq 16(%rdx),%rax // %rax is used as modifiable copy of %rdx cmpl $256,%esi je L$14rounds cmpl $192,%esi @@ -1986,97 +1986,97 @@ jne L$bad_keybits L$10rounds: - movl $9,%esi + movl $9,%esi // 10 rounds for 128-bit key - movups %xmm0,(%rdx) - aeskeygenassist $0x1,%xmm0,%xmm1 + movups %xmm0,(%rdx) // round 0 + aeskeygenassist $0x1,%xmm0,%xmm1 // round 1 call L$key_expansion_128_cold - aeskeygenassist $0x2,%xmm0,%xmm1 + aeskeygenassist $0x2,%xmm0,%xmm1 // round 2 call L$key_expansion_128 - aeskeygenassist $0x4,%xmm0,%xmm1 + aeskeygenassist $0x4,%xmm0,%xmm1 // round 3 call L$key_expansion_128 - aeskeygenassist $0x8,%xmm0,%xmm1 + aeskeygenassist $0x8,%xmm0,%xmm1 // round 4 call L$key_expansion_128 - aeskeygenassist $0x10,%xmm0,%xmm1 + aeskeygenassist $0x10,%xmm0,%xmm1 // round 5 call L$key_expansion_128 - aeskeygenassist $0x20,%xmm0,%xmm1 + aeskeygenassist $0x20,%xmm0,%xmm1 // round 6 call L$key_expansion_128 - aeskeygenassist $0x40,%xmm0,%xmm1 + aeskeygenassist $0x40,%xmm0,%xmm1 // round 7 call L$key_expansion_128 - aeskeygenassist $0x80,%xmm0,%xmm1 + aeskeygenassist $0x80,%xmm0,%xmm1 // round 8 call L$key_expansion_128 - aeskeygenassist $0x1b,%xmm0,%xmm1 + aeskeygenassist $0x1b,%xmm0,%xmm1 // round 9 call L$key_expansion_128 - aeskeygenassist $0x36,%xmm0,%xmm1 + aeskeygenassist $0x36,%xmm0,%xmm1 // round 10 call L$key_expansion_128 movups %xmm0,(%rax) - movl %esi,80(%rax) + movl %esi,80(%rax) // 240(%rdx) xorl %eax,%eax jmp L$enc_key_ret .p2align 4 L$12rounds: - movq 16(%rdi),%xmm2 - movl $11,%esi + movq 16(%rdi),%xmm2 // remaining 1/3 of *userKey + movl $11,%esi // 12 rounds for 192 - movups %xmm0,(%rdx) - aeskeygenassist $0x1,%xmm2,%xmm1 + movups %xmm0,(%rdx) // round 0 + aeskeygenassist $0x1,%xmm2,%xmm1 // round 1,2 call L$key_expansion_192a_cold - aeskeygenassist $0x2,%xmm2,%xmm1 + aeskeygenassist $0x2,%xmm2,%xmm1 // round 2,3 call L$key_expansion_192b - aeskeygenassist $0x4,%xmm2,%xmm1 + aeskeygenassist $0x4,%xmm2,%xmm1 // round 4,5 call L$key_expansion_192a - aeskeygenassist $0x8,%xmm2,%xmm1 + aeskeygenassist $0x8,%xmm2,%xmm1 // round 5,6 call L$key_expansion_192b - aeskeygenassist $0x10,%xmm2,%xmm1 + aeskeygenassist $0x10,%xmm2,%xmm1 // round 7,8 call L$key_expansion_192a - aeskeygenassist $0x20,%xmm2,%xmm1 + aeskeygenassist $0x20,%xmm2,%xmm1 // round 8,9 call L$key_expansion_192b - aeskeygenassist $0x40,%xmm2,%xmm1 + aeskeygenassist $0x40,%xmm2,%xmm1 // round 10,11 call L$key_expansion_192a - aeskeygenassist $0x80,%xmm2,%xmm1 + aeskeygenassist $0x80,%xmm2,%xmm1 // round 11,12 call L$key_expansion_192b movups %xmm0,(%rax) - movl %esi,48(%rax) + movl %esi,48(%rax) // 240(%rdx) xorq %rax,%rax jmp L$enc_key_ret .p2align 4 L$14rounds: - movups 16(%rdi),%xmm2 - movl $13,%esi + movups 16(%rdi),%xmm2 // remaining half of *userKey + movl $13,%esi // 14 rounds for 256 leaq 16(%rax),%rax - movups %xmm0,(%rdx) - movups %xmm2,16(%rdx) - aeskeygenassist $0x1,%xmm2,%xmm1 + movups %xmm0,(%rdx) // round 0 + movups %xmm2,16(%rdx) // round 1 + aeskeygenassist $0x1,%xmm2,%xmm1 // round 2 call L$key_expansion_256a_cold - aeskeygenassist $0x1,%xmm0,%xmm1 + aeskeygenassist $0x1,%xmm0,%xmm1 // round 3 call L$key_expansion_256b - aeskeygenassist $0x2,%xmm2,%xmm1 + aeskeygenassist $0x2,%xmm2,%xmm1 // round 4 call L$key_expansion_256a - aeskeygenassist $0x2,%xmm0,%xmm1 + aeskeygenassist $0x2,%xmm0,%xmm1 // round 5 call L$key_expansion_256b - aeskeygenassist $0x4,%xmm2,%xmm1 + aeskeygenassist $0x4,%xmm2,%xmm1 // round 6 call L$key_expansion_256a - aeskeygenassist $0x4,%xmm0,%xmm1 + aeskeygenassist $0x4,%xmm0,%xmm1 // round 7 call L$key_expansion_256b - aeskeygenassist $0x8,%xmm2,%xmm1 + aeskeygenassist $0x8,%xmm2,%xmm1 // round 8 call L$key_expansion_256a - aeskeygenassist $0x8,%xmm0,%xmm1 + aeskeygenassist $0x8,%xmm0,%xmm1 // round 9 call L$key_expansion_256b - aeskeygenassist $0x10,%xmm2,%xmm1 + aeskeygenassist $0x10,%xmm2,%xmm1 // round 10 call L$key_expansion_256a - aeskeygenassist $0x10,%xmm0,%xmm1 + aeskeygenassist $0x10,%xmm0,%xmm1 // round 11 call L$key_expansion_256b - aeskeygenassist $0x20,%xmm2,%xmm1 + aeskeygenassist $0x20,%xmm2,%xmm1 // round 12 call L$key_expansion_256a - aeskeygenassist $0x20,%xmm0,%xmm1 + aeskeygenassist $0x20,%xmm0,%xmm1 // round 13 call L$key_expansion_256b - aeskeygenassist $0x40,%xmm2,%xmm1 + aeskeygenassist $0x40,%xmm2,%xmm1 // round 14 call L$key_expansion_256a movups %xmm0,(%rax) - movl %esi,16(%rax) + movl %esi,16(%rax) // 240(%rdx) xorq %rax,%rax jmp L$enc_key_ret @@ -2107,7 +2107,7 @@ xorps %xmm4,%xmm0 shufps $140,%xmm0,%xmm4 xorps %xmm4,%xmm0 - shufps $255,%xmm1,%xmm1 + shufps $255,%xmm1,%xmm1 // critical path xorps %xmm1,%xmm0 ret @@ -2126,7 +2126,7 @@ shufps $140,%xmm0,%xmm4 pslldq $4,%xmm3 xorps %xmm4,%xmm0 - pshufd $85,%xmm1,%xmm1 + pshufd $85,%xmm1,%xmm1 // critical path pxor %xmm3,%xmm2 pxor %xmm1,%xmm0 pshufd $255,%xmm0,%xmm3 @@ -2156,7 +2156,7 @@ xorps %xmm4,%xmm0 shufps $140,%xmm0,%xmm4 xorps %xmm4,%xmm0 - shufps $255,%xmm1,%xmm1 + shufps $255,%xmm1,%xmm1 // critical path xorps %xmm1,%xmm0 ret @@ -2171,7 +2171,7 @@ xorps %xmm4,%xmm2 shufps $140,%xmm2,%xmm4 xorps %xmm4,%xmm2 - shufps $170,%xmm1,%xmm1 + shufps $170,%xmm1,%xmm1 // critical path xorps %xmm1,%xmm2 ret @@ -2192,9 +2192,9 @@ - movups (%rdi),%xmm0 - xorps %xmm4,%xmm4 - leaq 16(%rdx),%rax + movups (%rdi),%xmm0 // pull first 128 bits of *userKey + xorps %xmm4,%xmm4 // low dword of xmm4 is assumed 0 + leaq 16(%rdx),%rax // %rax is used as modifiable copy of %rdx cmpl $256,%esi je L$14rounds_alt cmpl $192,%esi @@ -2202,7 +2202,7 @@ cmpl $128,%esi jne L$bad_keybits_alt - movl $9,%esi + movl $9,%esi // 10 rounds for 128-bit key movdqa L$key_rotate(%rip),%xmm5 movl $8,%r10d movdqa L$key_rcon1(%rip),%xmm4 @@ -2264,14 +2264,14 @@ pxor %xmm2,%xmm0 movdqu %xmm0,16(%rax) - movl %esi,96(%rax) + movl %esi,96(%rax) // 240(%rdx) xorl %eax,%eax jmp L$enc_key_ret_alt .p2align 4 L$12rounds_alt: - movq 16(%rdi),%xmm2 - movl $11,%esi + movq 16(%rdi),%xmm2 // remaining 1/3 of *userKey + movl $11,%esi // 12 rounds for 192 movdqa L$key_rotate192(%rip),%xmm5 movdqa L$key_rcon1(%rip),%xmm4 movl $8,%r10d @@ -2307,14 +2307,14 @@ decl %r10d jnz L$oop_key192 - movl %esi,32(%rax) + movl %esi,32(%rax) // 240(%rdx) xorl %eax,%eax jmp L$enc_key_ret_alt .p2align 4 L$14rounds_alt: - movups 16(%rdi),%xmm2 - movl $13,%esi + movups 16(%rdi),%xmm2 // remaining half of *userKey + movl $13,%esi // 14 rounds for 256 leaq 16(%rax),%rax movdqa L$key_rotate(%rip),%xmm5 movdqa L$key_rcon1(%rip),%xmm4 @@ -2364,7 +2364,7 @@ jmp L$oop_key256 L$done_key256: - movl %esi,16(%rax) + movl %esi,16(%rax) // 240(%rdx) xorl %eax,%eax jmp L$enc_key_ret_alt
diff --git a/gen/bcm/aesni-x86_64-linux.S b/gen/bcm/aesni-x86_64-linux.S index 5093961..19996a7 100644 --- a/gen/bcm/aesni-x86_64-linux.S +++ b/gen/bcm/aesni-x86_64-linux.S
@@ -17,8 +17,8 @@ .hidden BORINGSSL_function_hit movb $1,BORINGSSL_function_hit+1(%rip) #endif - movups (%rdi),%xmm2 - movl 240(%rdx),%eax + movups (%rdi),%xmm2 // load input + movl 240(%rdx),%eax // key->rounds movups (%rdx),%xmm0 movups 16(%rdx),%xmm1 leaq 32(%rdx),%rdx @@ -28,11 +28,11 @@ decl %eax movups (%rdx),%xmm1 leaq 16(%rdx),%rdx - jnz .Loop_enc1_1 + jnz .Loop_enc1_1 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // output pxor %xmm2,%xmm2 ret .cfi_endproc @@ -45,8 +45,8 @@ aes_hw_decrypt: .cfi_startproc _CET_ENDBR - movups (%rdi),%xmm2 - movl 240(%rdx),%eax + movups (%rdi),%xmm2 // load input + movl 240(%rdx),%eax // key->rounds movups (%rdx),%xmm0 movups 16(%rdx),%xmm1 leaq 32(%rdx),%rdx @@ -56,11 +56,11 @@ decl %eax movups (%rdx),%xmm1 leaq 16(%rdx),%rdx - jnz .Loop_dec1_2 + jnz .Loop_dec1_2 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // output pxor %xmm2,%xmm2 ret .cfi_endproc @@ -76,7 +76,7 @@ xorps %xmm0,%xmm3 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax .Lenc_loop2: @@ -107,7 +107,7 @@ xorps %xmm0,%xmm3 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax .Ldec_loop2: @@ -139,7 +139,7 @@ xorps %xmm0,%xmm4 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax .Lenc_loop3: @@ -175,7 +175,7 @@ xorps %xmm0,%xmm4 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax addq $16,%rax .Ldec_loop3: @@ -212,7 +212,7 @@ xorps %xmm0,%xmm5 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax .byte 0x0f,0x1f,0x00 addq $16,%rax @@ -254,7 +254,7 @@ xorps %xmm0,%xmm5 movups 32(%rcx),%xmm0 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax .byte 0x0f,0x1f,0x00 addq $16,%rax @@ -295,7 +295,7 @@ pxor %xmm0,%xmm4 aesenc %xmm1,%xmm2 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesenc %xmm1,%xmm3 pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 @@ -351,7 +351,7 @@ pxor %xmm0,%xmm4 aesdec %xmm1,%xmm2 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesdec %xmm1,%xmm3 pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 @@ -408,7 +408,7 @@ pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesenc %xmm1,%xmm2 pxor %xmm0,%xmm7 pxor %xmm0,%xmm8 @@ -474,7 +474,7 @@ pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 leaq 32(%rcx,%rax,1),%rcx - negq %rax + negq %rax // %eax aesdec %xmm1,%xmm2 pxor %xmm0,%xmm7 pxor %xmm0,%xmm8 @@ -534,20 +534,20 @@ aes_hw_ecb_encrypt: .cfi_startproc _CET_ENDBR - andq $-16,%rdx - jz .Lecb_ret + andq $-16,%rdx // if (%rdx<16) + jz .Lecb_ret // return - movl 240(%rcx),%eax + movl 240(%rcx),%eax // key->rounds movups (%rcx),%xmm0 - movq %rcx,%r11 - movl %eax,%r10d - testl %r8d,%r8d + movq %rcx,%r11 // backup %rcx + movl %eax,%r10d // backup %eax + testl %r8d,%r8d // 5th argument jz .Lecb_decrypt +// --------------------------- ECB ENCRYPT ------------------------------ + cmpq $0x80,%rdx // if (%rdx<8*16) + jb .Lecb_enc_tail // short input - cmpq $0x80,%rdx - jb .Lecb_enc_tail - - movdqu (%rdi),%xmm2 + movdqu (%rdi),%xmm2 // load 8 input blocks movdqu 16(%rdi),%xmm3 movdqu 32(%rdi),%xmm4 movdqu 48(%rdi),%xmm5 @@ -555,15 +555,15 @@ movdqu 80(%rdi),%xmm7 movdqu 96(%rdi),%xmm8 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi - subq $0x80,%rdx + leaq 128(%rdi),%rdi // %rdi+=8*16 + subq $0x80,%rdx // %rdx-=8*16 (can be zero) jmp .Lecb_enc_loop8_enter .align 16 .Lecb_enc_loop8: - movups %xmm2,(%rsi) - movq %r11,%rcx - movdqu (%rdi),%xmm2 - movl %r10d,%eax + movups %xmm2,(%rsi) // store 8 output blocks + movq %r11,%rcx // restore %rcx + movdqu (%rdi),%xmm2 // load 8 input blocks + movl %r10d,%eax // restore %eax movups %xmm3,16(%rsi) movdqu 16(%rdi),%xmm3 movups %xmm4,32(%rsi) @@ -577,31 +577,31 @@ movups %xmm8,96(%rsi) movdqu 96(%rdi),%xmm8 movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // %rsi+=8*16 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // %rdi+=8*16 .Lecb_enc_loop8_enter: call _aesni_encrypt8 subq $0x80,%rdx - jnc .Lecb_enc_loop8 + jnc .Lecb_enc_loop8 // loop if %rdx-=8*16 didn't borrow - movups %xmm2,(%rsi) - movq %r11,%rcx + movups %xmm2,(%rsi) // store 8 output blocks + movq %r11,%rcx // restore %rcx movups %xmm3,16(%rsi) - movl %r10d,%eax + movl %r10d,%eax // restore %eax movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) movups %xmm6,64(%rsi) movups %xmm7,80(%rsi) movups %xmm8,96(%rsi) movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi - addq $0x80,%rdx - jz .Lecb_ret + leaq 128(%rsi),%rsi // %rsi+=8*16 + addq $0x80,%rdx // restore real remaining %rdx + jz .Lecb_ret // done if (%rdx==0) -.Lecb_enc_tail: +.Lecb_enc_tail: // %rdx is less than 8*16 movups (%rdi),%xmm2 cmpq $0x20,%rdx jb .Lecb_enc_one @@ -620,7 +620,7 @@ movdqu 96(%rdi),%xmm8 xorps %xmm9,%xmm9 call _aesni_encrypt8 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 7 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) @@ -639,27 +639,27 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz .Loop_enc1_3 + jnz .Loop_enc1_3 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store one output block jmp .Lecb_ret .align 16 .Lecb_enc_two: call _aesni_encrypt2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 2 output blocks movups %xmm3,16(%rsi) jmp .Lecb_ret .align 16 .Lecb_enc_three: call _aesni_encrypt3 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 3 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) jmp .Lecb_ret .align 16 .Lecb_enc_four: call _aesni_encrypt4 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 4 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) @@ -668,7 +668,7 @@ .Lecb_enc_five: xorps %xmm7,%xmm7 call _aesni_encrypt6 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 5 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) @@ -677,20 +677,20 @@ .align 16 .Lecb_enc_six: call _aesni_encrypt6 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 6 output blocks movups %xmm3,16(%rsi) movups %xmm4,32(%rsi) movups %xmm5,48(%rsi) movups %xmm6,64(%rsi) movups %xmm7,80(%rsi) jmp .Lecb_ret - +// --------------------------- ECB DECRYPT ------------------------------# .align 16 .Lecb_decrypt: - cmpq $0x80,%rdx - jb .Lecb_dec_tail + cmpq $0x80,%rdx // if (%rdx<8*16) + jb .Lecb_dec_tail // short input - movdqu (%rdi),%xmm2 + movdqu (%rdi),%xmm2 // load 8 input blocks movdqu 16(%rdi),%xmm3 movdqu 32(%rdi),%xmm4 movdqu 48(%rdi),%xmm5 @@ -698,15 +698,15 @@ movdqu 80(%rdi),%xmm7 movdqu 96(%rdi),%xmm8 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi - subq $0x80,%rdx + leaq 128(%rdi),%rdi // %rdi+=8*16 + subq $0x80,%rdx // %rdx-=8*16 (can be zero) jmp .Lecb_dec_loop8_enter .align 16 .Lecb_dec_loop8: - movups %xmm2,(%rsi) - movq %r11,%rcx - movdqu (%rdi),%xmm2 - movl %r10d,%eax + movups %xmm2,(%rsi) // store 8 output blocks + movq %r11,%rcx // restore %rcx + movdqu (%rdi),%xmm2 // load 8 input blocks + movl %r10d,%eax // restore %eax movups %xmm3,16(%rsi) movdqu 16(%rdi),%xmm3 movups %xmm4,32(%rsi) @@ -720,23 +720,23 @@ movups %xmm8,96(%rsi) movdqu 96(%rdi),%xmm8 movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // %rsi+=8*16 movdqu 112(%rdi),%xmm9 - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // %rdi+=8*16 .Lecb_dec_loop8_enter: call _aesni_decrypt8 movups (%r11),%xmm0 subq $0x80,%rdx - jnc .Lecb_dec_loop8 + jnc .Lecb_dec_loop8 // loop if %rdx-=8*16 didn't borrow - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 - movq %r11,%rcx + movups %xmm2,(%rsi) // store 8 output blocks + pxor %xmm2,%xmm2 // clear register bank + movq %r11,%rcx // restore %rcx movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 - movl %r10d,%eax + movl %r10d,%eax // restore %eax movups %xmm4,32(%rsi) pxor %xmm4,%xmm4 movups %xmm5,48(%rsi) @@ -749,9 +749,9 @@ pxor %xmm8,%xmm8 movups %xmm9,112(%rsi) pxor %xmm9,%xmm9 - leaq 128(%rsi),%rsi - addq $0x80,%rdx - jz .Lecb_ret + leaq 128(%rsi),%rsi // %rsi+=8*16 + addq $0x80,%rdx // restore real remaining %rdx + jz .Lecb_ret // done if (%rdx==0) .Lecb_dec_tail: movups (%rdi),%xmm2 @@ -773,8 +773,8 @@ movups (%rcx),%xmm0 xorps %xmm9,%xmm9 call _aesni_decrypt8 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 7 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -800,24 +800,24 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz .Loop_dec1_4 + jnz .Loop_dec1_4 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store one output block + pxor %xmm2,%xmm2 // clear register bank jmp .Lecb_ret .align 16 .Lecb_dec_two: call _aesni_decrypt2 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 2 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 jmp .Lecb_ret .align 16 .Lecb_dec_three: call _aesni_decrypt3 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 3 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -826,8 +826,8 @@ .align 16 .Lecb_dec_four: call _aesni_decrypt4 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 4 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -839,8 +839,8 @@ .Lecb_dec_five: xorps %xmm7,%xmm7 call _aesni_decrypt6 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 5 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -854,8 +854,8 @@ .align 16 .Lecb_dec_six: call _aesni_decrypt6 - movups %xmm2,(%rsi) - pxor %xmm2,%xmm2 + movups %xmm2,(%rsi) // store 6 output blocks + pxor %xmm2,%xmm2 // clear register bank movups %xmm3,16(%rsi) pxor %xmm3,%xmm3 movups %xmm4,32(%rsi) @@ -868,7 +868,7 @@ pxor %xmm7,%xmm7 .Lecb_ret: - xorps %xmm0,%xmm0 + xorps %xmm0,%xmm0 // %xmm0 pxor %xmm1,%xmm1 ret .cfi_endproc @@ -886,11 +886,11 @@ cmpq $1,%rdx jne .Lctr32_bulk - - +// handle single block without allocating stack frame, +// useful when handling edges movups (%r8),%xmm2 movups (%rdi),%xmm3 - movl 240(%rcx),%edx + movl 240(%rcx),%edx // key->rounds movups (%rcx),%xmm0 movups 16(%rcx),%xmm1 leaq 32(%rcx),%rcx @@ -900,9 +900,9 @@ decl %edx movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz .Loop_enc1_5 + jnz .Loop_enc1_5 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 xorps %xmm3,%xmm2 pxor %xmm3,%xmm3 @@ -912,22 +912,22 @@ .align 16 .Lctr32_bulk: - leaq (%rsp),%r11 + leaq (%rsp),%r11 // use %r11 as frame pointer .cfi_def_cfa_register %r11 pushq %rbp .cfi_offset %rbp,-16 subq $128,%rsp - andq $-16,%rsp + andq $-16,%rsp // Linux kernel stack can be incorrectly seeded - - +// 8 16-byte words on top of stack are counter values +// xor-ed with zero-round key movdqu (%r8),%xmm2 movdqu (%rcx),%xmm0 - movl 12(%r8),%r8d + movl 12(%r8),%r8d // counter LSB pxor %xmm0,%xmm2 - movl 12(%rcx),%ebp - movdqa %xmm2,0(%rsp) + movl 12(%rcx),%ebp // 0-round key LSB + movdqa %xmm2,0(%rsp) // populate counter block bswapl %r8d movdqa %xmm2,%xmm3 movdqa %xmm2,%xmm4 @@ -935,7 +935,7 @@ movdqa %xmm2,64(%rsp) movdqa %xmm2,80(%rsp) movdqa %xmm2,96(%rsp) - movq %rdx,%r10 + movq %rdx,%r10 // about to borrow %rdx movdqa %xmm2,112(%rsp) leaq 1(%r8),%rax @@ -949,7 +949,7 @@ movdqa %xmm3,16(%rsp) pinsrd $3,%edx,%xmm4 bswapl %eax - movq %r10,%rdx + movq %r10,%rdx // restore %rdx leaq 4(%r8),%r10 movdqa %xmm4,32(%rsp) xorl %ebp,%eax @@ -961,7 +961,7 @@ movl %r10d,64+12(%rsp) bswapl %r9d leaq 6(%r8),%r10 - movl 240(%rcx),%eax + movl 240(%rcx),%eax // key->rounds xorl %ebp,%r9d bswapl %r10d movl %r9d,80+12(%rsp) @@ -977,16 +977,16 @@ movdqa 64(%rsp),%xmm6 movdqa 80(%rsp),%xmm7 - cmpq $8,%rdx - jb .Lctr32_tail + cmpq $8,%rdx // %rdx is in blocks + jb .Lctr32_tail // short input if (%rdx<8) - leaq 128(%rcx),%rcx - subq $8,%rdx + leaq 128(%rcx),%rcx // size optimization + subq $8,%rdx // %rdx is biased by -8 jmp .Lctr32_loop8 .align 32 .Lctr32_loop8: - addl $8,%r8d + addl $8,%r8d // next counter value movdqa 96(%rsp),%xmm8 aesenc %xmm1,%xmm2 movl %r8d,%r9d @@ -998,7 +998,7 @@ xorl %ebp,%r9d nop aesenc %xmm1,%xmm5 - movl %r9d,0+12(%rsp) + movl %r9d,0+12(%rsp) // store next counter value leaq 1(%r8),%r9 aesenc %xmm1,%xmm6 aesenc %xmm1,%xmm7 @@ -1094,7 +1094,7 @@ aesenc %xmm0,%xmm3 aesenc %xmm0,%xmm4 xorl %ebp,%r9d - movdqu 0(%rdi),%xmm10 + movdqu 0(%rdi),%xmm10 // start loading input aesenc %xmm0,%xmm5 movl %r9d,112+12(%rsp) cmpl $11,%eax @@ -1151,7 +1151,7 @@ .align 16 .Lctr32_enc_done: movdqu 16(%rdi),%xmm11 - pxor %xmm0,%xmm10 + pxor %xmm0,%xmm10 // input^=round[last] movdqu 32(%rdi),%xmm12 pxor %xmm0,%xmm11 movdqu 48(%rdi),%xmm13 @@ -1160,8 +1160,8 @@ pxor %xmm0,%xmm13 movdqu 80(%rdi),%xmm15 pxor %xmm0,%xmm14 - prefetcht0 448(%rdi) - prefetcht0 512(%rdi) + prefetcht0 448(%rdi) // We process 128 bytes (8*16), so to prefetch 1 iteration + prefetcht0 512(%rdi) // We need to prefetch 2 64 byte lines pxor %xmm0,%xmm15 aesenc %xmm1,%xmm2 aesenc %xmm1,%xmm3 @@ -1171,15 +1171,15 @@ aesenc %xmm1,%xmm7 aesenc %xmm1,%xmm8 aesenc %xmm1,%xmm9 - movdqu 96(%rdi),%xmm1 - leaq 128(%rdi),%rdi + movdqu 96(%rdi),%xmm1 // borrow %xmm1 for inp[6] + leaq 128(%rdi),%rdi // %rdi+=8*16 - aesenclast %xmm10,%xmm2 - pxor %xmm0,%xmm1 + aesenclast %xmm10,%xmm2 // is inp[N]^round[last] + pxor %xmm0,%xmm1 // borrowed movdqu 112-128(%rdi),%xmm10 aesenclast %xmm11,%xmm3 pxor %xmm0,%xmm10 - movdqa 0(%rsp),%xmm11 + movdqa 0(%rsp),%xmm11 // load next counter block aesenclast %xmm12,%xmm4 aesenclast %xmm13,%xmm5 movdqa 16(%rsp),%xmm12 @@ -1190,10 +1190,10 @@ movdqa 64(%rsp),%xmm15 aesenclast %xmm1,%xmm8 movdqa 80(%rsp),%xmm0 - movups 16-128(%rcx),%xmm1 + movups 16-128(%rcx),%xmm1 // real 1st-round key aesenclast %xmm10,%xmm9 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store 8 output blocks movdqa %xmm11,%xmm2 movups %xmm3,16(%rsi) movdqa %xmm12,%xmm3 @@ -1207,24 +1207,24 @@ movdqa %xmm0,%xmm7 movups %xmm8,96(%rsi) movups %xmm9,112(%rsi) - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // %rsi+=8*16 subq $8,%rdx - jnc .Lctr32_loop8 + jnc .Lctr32_loop8 // loop if %rdx-=8 didn't borrow - addq $8,%rdx - jz .Lctr32_done + addq $8,%rdx // restore real remaining %rdx + jz .Lctr32_done // done if (%rdx==0) leaq -128(%rcx),%rcx .Lctr32_tail: - - +// note that at this point %xmm2..5 are populated with +// counter values xor-ed with 0-round key leaq 16(%rcx),%rcx cmpq $4,%rdx jb .Lctr32_loop3 je .Lctr32_loop4 - +// if (%rdx>4) compute 7 E(counter) shll $4,%eax movdqa 96(%rsp),%xmm8 pxor %xmm9,%xmm9 @@ -1232,14 +1232,14 @@ movups 16(%rcx),%xmm0 aesenc %xmm1,%xmm2 aesenc %xmm1,%xmm3 - leaq 32-16(%rcx,%rax,1),%rcx + leaq 32-16(%rcx,%rax,1),%rcx // prepare for .Lenc_loop8_enter negq %rax aesenc %xmm1,%xmm4 - addq $16,%rax + addq $16,%rax // prepare for .Lenc_loop8_enter movups (%rdi),%xmm10 aesenc %xmm1,%xmm5 aesenc %xmm1,%xmm6 - movups 16(%rdi),%xmm11 + movups 16(%rdi),%xmm11 // pre-load input movups 32(%rdi),%xmm12 aesenc %xmm1,%xmm7 aesenc %xmm1,%xmm8 @@ -1250,7 +1250,7 @@ pxor %xmm10,%xmm2 movdqu 64(%rdi),%xmm10 pxor %xmm11,%xmm3 - movdqu %xmm2,(%rsi) + movdqu %xmm2,(%rsi) // store output pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) pxor %xmm13,%xmm5 @@ -1259,17 +1259,17 @@ movdqu %xmm5,48(%rsi) movdqu %xmm6,64(%rsi) cmpq $6,%rdx - jb .Lctr32_done + jb .Lctr32_done // %rdx was 5, stop store movups 80(%rdi),%xmm11 xorps %xmm11,%xmm7 movups %xmm7,80(%rsi) - je .Lctr32_done + je .Lctr32_done // %rdx was 6, stop store movups 96(%rdi),%xmm12 xorps %xmm12,%xmm8 movups %xmm8,96(%rsi) - jmp .Lctr32_done + jmp .Lctr32_done // %rdx was 7, stop store .align 32 .Lctr32_loop4: @@ -1283,7 +1283,7 @@ jnz .Lctr32_loop4 aesenclast %xmm1,%xmm2 aesenclast %xmm1,%xmm3 - movups (%rdi),%xmm10 + movups (%rdi),%xmm10 // load input movups 16(%rdi),%xmm11 aesenclast %xmm1,%xmm4 aesenclast %xmm1,%xmm5 @@ -1291,14 +1291,14 @@ movups 48(%rdi),%xmm13 xorps %xmm10,%xmm2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output xorps %xmm11,%xmm3 movups %xmm3,16(%rsi) pxor %xmm12,%xmm4 movdqu %xmm4,32(%rsi) pxor %xmm13,%xmm5 movdqu %xmm5,48(%rsi) - jmp .Lctr32_done + jmp .Lctr32_done // %rdx was 4, stop store .align 32 .Lctr32_loop3: @@ -1313,23 +1313,23 @@ aesenclast %xmm1,%xmm3 aesenclast %xmm1,%xmm4 - movups (%rdi),%xmm10 + movups (%rdi),%xmm10 // load input xorps %xmm10,%xmm2 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output cmpq $2,%rdx - jb .Lctr32_done + jb .Lctr32_done // %rdx was 1, stop store movups 16(%rdi),%xmm11 xorps %xmm11,%xmm3 movups %xmm3,16(%rsi) - je .Lctr32_done + je .Lctr32_done // %rdx was 2, stop store movups 32(%rdi),%xmm12 xorps %xmm12,%xmm4 - movups %xmm4,32(%rsi) + movups %xmm4,32(%rsi) // %rdx was 3, stop store .Lctr32_done: - xorps %xmm0,%xmm0 + xorps %xmm0,%xmm0 // clear register bank xorl %ebp,%ebp pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -1338,7 +1338,7 @@ pxor %xmm5,%xmm5 pxor %xmm6,%xmm6 pxor %xmm7,%xmm7 - movaps %xmm0,0(%rsp) + movaps %xmm0,0(%rsp) // clear stack pxor %xmm8,%xmm8 movaps %xmm0,16(%rsp) pxor %xmm9,%xmm9 @@ -1369,15 +1369,15 @@ aes_hw_cbc_encrypt: .cfi_startproc _CET_ENDBR - testq %rdx,%rdx + testq %rdx,%rdx // check length jz .Lcbc_ret - movl 240(%rcx),%r10d - movq %rcx,%r11 - testl %r9d,%r9d + movl 240(%rcx),%r10d // key->rounds + movq %rcx,%r11 // backup %rcx + testl %r9d,%r9d // 6th argument jz .Lcbc_decrypt - - movups (%r8),%xmm2 +// --------------------------- CBC ENCRYPT ------------------------------ + movups (%r8),%xmm2 // load iv as initial state movl %r10d,%eax cmpq $16,%rdx jb .Lcbc_enc_tail @@ -1385,9 +1385,9 @@ jmp .Lcbc_enc_loop .align 16 .Lcbc_enc_loop: - movups (%rdi),%xmm3 + movups (%rdi),%xmm3 // load input leaq 16(%rdi),%rdi - +// xorps %xmm3,%xmm2 movups (%rcx),%xmm0 movups 16(%rcx),%xmm1 xorps %xmm0,%xmm3 @@ -1398,17 +1398,17 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz .Loop_enc1_6 + jnz .Loop_enc1_6 // loop body is 16 bytes aesenclast %xmm1,%xmm2 - movl %r10d,%eax - movq %r11,%rcx - movups %xmm2,0(%rsi) + movl %r10d,%eax // restore %eax + movq %r11,%rcx // restore %rcx + movups %xmm2,0(%rsi) // store output leaq 16(%rsi),%rsi subq $16,%rdx jnc .Lcbc_enc_loop addq $16,%rdx jnz .Lcbc_enc_tail - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 movups %xmm2,(%r8) pxor %xmm2,%xmm2 @@ -1416,30 +1416,30 @@ jmp .Lcbc_ret .Lcbc_enc_tail: - movq %rdx,%rcx - xchgq %rdi,%rsi -.long 0x9066A4F3 - movl $16,%ecx + movq %rdx,%rcx // zaps %rcx + xchgq %rdi,%rsi // %rdi is %rsi and %rsi is %rdi now +.long 0x9066A4F3 // rep movsb + movl $16,%ecx // zero tail subq %rdx,%rcx xorl %eax,%eax -.long 0x9066AAF3 - leaq -16(%rdi),%rdi - movl %r10d,%eax - movq %rdi,%rsi - movq %r11,%rcx - xorq %rdx,%rdx - jmp .Lcbc_enc_loop - +.long 0x9066AAF3 // rep stosb + leaq -16(%rdi),%rdi // rewind %rsi by 1 block + movl %r10d,%eax // restore %eax + movq %rdi,%rsi // %rdi and %rsi are the same + movq %r11,%rcx // restore %rcx + xorq %rdx,%rdx // len=16 + jmp .Lcbc_enc_loop // one more spin +// --------------------------- CBC DECRYPT ------------------------------# .align 16 .Lcbc_decrypt: cmpq $16,%rdx jne .Lcbc_decrypt_bulk - - - movdqu (%rdi),%xmm2 - movdqu (%r8),%xmm3 - movdqa %xmm2,%xmm4 +// handle single block without allocating stack frame, +// useful in ciphertext stealing mode + movdqu (%rdi),%xmm2 // load input + movdqu (%r8),%xmm3 // load iv + movdqa %xmm2,%xmm4 // future iv movups (%rcx),%xmm0 movups 16(%rcx),%xmm1 leaq 32(%rcx),%rcx @@ -1449,32 +1449,32 @@ decl %r10d movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz .Loop_dec1_7 + jnz .Loop_dec1_7 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 - pxor %xmm0,%xmm0 + pxor %xmm0,%xmm0 // clear register bank pxor %xmm1,%xmm1 - movdqu %xmm4,(%r8) - xorps %xmm3,%xmm2 + movdqu %xmm4,(%r8) // store iv + xorps %xmm3,%xmm2 // ^=iv pxor %xmm3,%xmm3 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output pxor %xmm2,%xmm2 jmp .Lcbc_ret .align 16 .Lcbc_decrypt_bulk: - leaq (%rsp),%r11 + leaq (%rsp),%r11 // frame pointer .cfi_def_cfa_register %r11 pushq %rbp .cfi_offset %rbp,-16 subq $16,%rsp - andq $-16,%rsp - movq %rcx,%rbp + andq $-16,%rsp // Linux kernel stack can be incorrectly seeded + movq %rcx,%rbp // [re-]backup %rcx [after reassignment] movups (%r8),%xmm10 movl %r10d,%eax cmpq $0x50,%rdx jbe .Lcbc_dec_tail movups (%rcx),%xmm0 - movdqu 0(%rdi),%xmm2 + movdqu 0(%rdi),%xmm2 // load input movdqu 16(%rdi),%xmm3 movdqa %xmm2,%xmm11 movdqu 32(%rdi),%xmm4 @@ -1488,8 +1488,8 @@ cmpq $0x70,%rdx jbe .Lcbc_dec_six_or_seven - subq $0x70,%rdx - leaq 112(%rcx),%rcx + subq $0x70,%rdx // %rdx is biased by -7*16 + leaq 112(%rcx),%rcx // size optimization jmp .Lcbc_dec_loop8_enter .align 16 .Lcbc_dec_loop8: @@ -1503,7 +1503,7 @@ movups 16-112(%rcx),%xmm1 pxor %xmm0,%xmm4 movq $-1,%rbp - cmpq $0x70,%rdx + cmpq $0x70,%rdx // is there at least 0x60 bytes ahead? pxor %xmm0,%xmm5 pxor %xmm0,%xmm6 pxor %xmm0,%xmm7 @@ -1652,11 +1652,11 @@ movdqu 80(%rdi),%xmm1 aesdeclast %xmm10,%xmm2 - movdqu 96(%rdi),%xmm10 + movdqu 96(%rdi),%xmm10 // borrow %xmm10 pxor %xmm0,%xmm1 aesdeclast %xmm11,%xmm3 pxor %xmm0,%xmm10 - movdqu 112(%rdi),%xmm0 + movdqu 112(%rdi),%xmm0 // next IV aesdeclast %xmm12,%xmm4 leaq 128(%rdi),%rdi movdqu 0(%rbp),%xmm11 @@ -1669,11 +1669,11 @@ movdqu 48(%rbp),%xmm14 movdqu 64(%rbp),%xmm15 aesdeclast %xmm10,%xmm9 - movdqa %xmm0,%xmm10 + movdqa %xmm0,%xmm10 // return %xmm10 movdqu 80(%rbp),%xmm1 movups -112(%rcx),%xmm0 - movups %xmm2,(%rsi) + movups %xmm2,(%rsi) // store output movdqa %xmm11,%xmm2 movups %xmm3,16(%rsi) movdqa %xmm12,%xmm3 @@ -1707,13 +1707,13 @@ movaps %xmm7,%xmm8 call _aesni_decrypt6 - pxor %xmm10,%xmm2 + pxor %xmm10,%xmm2 // ^= IV movaps %xmm8,%xmm10 pxor %xmm11,%xmm3 movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1734,13 +1734,13 @@ xorps %xmm9,%xmm9 call _aesni_decrypt8 movups 80(%rdi),%xmm9 - pxor %xmm10,%xmm2 + pxor %xmm10,%xmm2 // ^= IV movups 96(%rdi),%xmm10 pxor %xmm11,%xmm3 movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1762,24 +1762,24 @@ .Lcbc_dec_tail: movups (%rdi),%xmm2 subq $0x10,%rdx - jbe .Lcbc_dec_one + jbe .Lcbc_dec_one // %rdx is 1*16 or less movups 16(%rdi),%xmm3 movaps %xmm2,%xmm11 subq $0x10,%rdx - jbe .Lcbc_dec_two + jbe .Lcbc_dec_two // %rdx is 2*16 or less movups 32(%rdi),%xmm4 movaps %xmm3,%xmm12 subq $0x10,%rdx - jbe .Lcbc_dec_three + jbe .Lcbc_dec_three // %rdx is 3*16 or less movups 48(%rdi),%xmm5 movaps %xmm4,%xmm13 subq $0x10,%rdx - jbe .Lcbc_dec_four + jbe .Lcbc_dec_four // %rdx is 4*16 or less - movups 64(%rdi),%xmm6 + movups 64(%rdi),%xmm6 // %rdx is 5*16 or less movaps %xmm5,%xmm14 movaps %xmm6,%xmm15 xorps %xmm7,%xmm7 @@ -1790,7 +1790,7 @@ movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1816,7 +1816,7 @@ decl %eax movups (%rcx),%xmm1 leaq 16(%rcx),%rcx - jnz .Loop_dec1_8 + jnz .Loop_dec1_8 // loop body is 16 bytes aesdeclast %xmm1,%xmm2 xorps %xmm10,%xmm2 movaps %xmm11,%xmm10 @@ -1830,7 +1830,7 @@ pxor %xmm11,%xmm3 movdqu %xmm2,(%rsi) movdqa %xmm3,%xmm2 - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank leaq 16(%rsi),%rsi jmp .Lcbc_dec_tail_collected .align 16 @@ -1843,7 +1843,7 @@ movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank movdqa %xmm4,%xmm2 pxor %xmm4,%xmm4 leaq 32(%rsi),%rsi @@ -1858,7 +1858,7 @@ movdqu %xmm2,(%rsi) pxor %xmm12,%xmm4 movdqu %xmm3,16(%rsi) - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm13,%xmm5 movdqu %xmm4,32(%rsi) pxor %xmm4,%xmm4 @@ -1869,10 +1869,10 @@ .align 16 .Lcbc_dec_clear_tail_collected: - pxor %xmm3,%xmm3 + pxor %xmm3,%xmm3 // clear register bank pxor %xmm4,%xmm4 pxor %xmm5,%xmm5 - pxor %xmm6,%xmm6 + pxor %xmm6,%xmm6 // %xmm6..9 pxor %xmm7,%xmm7 pxor %xmm8,%xmm8 pxor %xmm9,%xmm9 @@ -1891,11 +1891,11 @@ movq %rsi,%rdi subq %rdx,%rcx leaq (%rsp),%rsi -.long 0x9066A4F3 +.long 0x9066A4F3 // rep movsb movdqa %xmm2,(%rsp) .Lcbc_dec_ret: - xorps %xmm0,%xmm0 + xorps %xmm0,%xmm0 // %xmm0 pxor %xmm1,%xmm1 movq -8(%r11),%rbp .cfi_restore %rbp @@ -1916,9 +1916,9 @@ movl 240(%rdi),%esi shll $4,%esi - leaq 16(%rdi,%rsi,1),%rdx + leaq 16(%rdi,%rsi,1),%rdx // points at the end of key schedule - movups (%rdi),%xmm0 + movups (%rdi),%xmm0 // just swap movups (%rdx),%xmm1 movups %xmm0,(%rdx) movups %xmm1,(%rdi) @@ -1926,7 +1926,7 @@ leaq -16(%rdx),%rdx .Ldec_key_inverse: - movups (%rdi),%xmm0 + movups (%rdi),%xmm0 // swap and inverse movups (%rdx),%xmm1 aesimc %xmm0,%xmm0 aesimc %xmm1,%xmm1 @@ -1937,7 +1937,7 @@ cmpq %rdi,%rdx ja .Ldec_key_inverse - movups (%rdi),%xmm0 + movups (%rdi),%xmm0 // inverse middle aesimc %xmm0,%xmm0 pxor %xmm1,%xmm1 movups %xmm0,(%rdx) @@ -1960,9 +1960,9 @@ .cfi_adjust_cfa_offset 8 - movups (%rdi),%xmm0 - xorps %xmm4,%xmm4 - leaq 16(%rdx),%rax + movups (%rdi),%xmm0 // pull first 128 bits of *userKey + xorps %xmm4,%xmm4 // low dword of xmm4 is assumed 0 + leaq 16(%rdx),%rax // %rax is used as modifiable copy of %rdx cmpl $256,%esi je .L14rounds cmpl $192,%esi @@ -1971,97 +1971,97 @@ jne .Lbad_keybits .L10rounds: - movl $9,%esi + movl $9,%esi // 10 rounds for 128-bit key - movups %xmm0,(%rdx) - aeskeygenassist $0x1,%xmm0,%xmm1 + movups %xmm0,(%rdx) // round 0 + aeskeygenassist $0x1,%xmm0,%xmm1 // round 1 call .Lkey_expansion_128_cold - aeskeygenassist $0x2,%xmm0,%xmm1 + aeskeygenassist $0x2,%xmm0,%xmm1 // round 2 call .Lkey_expansion_128 - aeskeygenassist $0x4,%xmm0,%xmm1 + aeskeygenassist $0x4,%xmm0,%xmm1 // round 3 call .Lkey_expansion_128 - aeskeygenassist $0x8,%xmm0,%xmm1 + aeskeygenassist $0x8,%xmm0,%xmm1 // round 4 call .Lkey_expansion_128 - aeskeygenassist $0x10,%xmm0,%xmm1 + aeskeygenassist $0x10,%xmm0,%xmm1 // round 5 call .Lkey_expansion_128 - aeskeygenassist $0x20,%xmm0,%xmm1 + aeskeygenassist $0x20,%xmm0,%xmm1 // round 6 call .Lkey_expansion_128 - aeskeygenassist $0x40,%xmm0,%xmm1 + aeskeygenassist $0x40,%xmm0,%xmm1 // round 7 call .Lkey_expansion_128 - aeskeygenassist $0x80,%xmm0,%xmm1 + aeskeygenassist $0x80,%xmm0,%xmm1 // round 8 call .Lkey_expansion_128 - aeskeygenassist $0x1b,%xmm0,%xmm1 + aeskeygenassist $0x1b,%xmm0,%xmm1 // round 9 call .Lkey_expansion_128 - aeskeygenassist $0x36,%xmm0,%xmm1 + aeskeygenassist $0x36,%xmm0,%xmm1 // round 10 call .Lkey_expansion_128 movups %xmm0,(%rax) - movl %esi,80(%rax) + movl %esi,80(%rax) // 240(%rdx) xorl %eax,%eax jmp .Lenc_key_ret .align 16 .L12rounds: - movq 16(%rdi),%xmm2 - movl $11,%esi + movq 16(%rdi),%xmm2 // remaining 1/3 of *userKey + movl $11,%esi // 12 rounds for 192 - movups %xmm0,(%rdx) - aeskeygenassist $0x1,%xmm2,%xmm1 + movups %xmm0,(%rdx) // round 0 + aeskeygenassist $0x1,%xmm2,%xmm1 // round 1,2 call .Lkey_expansion_192a_cold - aeskeygenassist $0x2,%xmm2,%xmm1 + aeskeygenassist $0x2,%xmm2,%xmm1 // round 2,3 call .Lkey_expansion_192b - aeskeygenassist $0x4,%xmm2,%xmm1 + aeskeygenassist $0x4,%xmm2,%xmm1 // round 4,5 call .Lkey_expansion_192a - aeskeygenassist $0x8,%xmm2,%xmm1 + aeskeygenassist $0x8,%xmm2,%xmm1 // round 5,6 call .Lkey_expansion_192b - aeskeygenassist $0x10,%xmm2,%xmm1 + aeskeygenassist $0x10,%xmm2,%xmm1 // round 7,8 call .Lkey_expansion_192a - aeskeygenassist $0x20,%xmm2,%xmm1 + aeskeygenassist $0x20,%xmm2,%xmm1 // round 8,9 call .Lkey_expansion_192b - aeskeygenassist $0x40,%xmm2,%xmm1 + aeskeygenassist $0x40,%xmm2,%xmm1 // round 10,11 call .Lkey_expansion_192a - aeskeygenassist $0x80,%xmm2,%xmm1 + aeskeygenassist $0x80,%xmm2,%xmm1 // round 11,12 call .Lkey_expansion_192b movups %xmm0,(%rax) - movl %esi,48(%rax) + movl %esi,48(%rax) // 240(%rdx) xorq %rax,%rax jmp .Lenc_key_ret .align 16 .L14rounds: - movups 16(%rdi),%xmm2 - movl $13,%esi + movups 16(%rdi),%xmm2 // remaining half of *userKey + movl $13,%esi // 14 rounds for 256 leaq 16(%rax),%rax - movups %xmm0,(%rdx) - movups %xmm2,16(%rdx) - aeskeygenassist $0x1,%xmm2,%xmm1 + movups %xmm0,(%rdx) // round 0 + movups %xmm2,16(%rdx) // round 1 + aeskeygenassist $0x1,%xmm2,%xmm1 // round 2 call .Lkey_expansion_256a_cold - aeskeygenassist $0x1,%xmm0,%xmm1 + aeskeygenassist $0x1,%xmm0,%xmm1 // round 3 call .Lkey_expansion_256b - aeskeygenassist $0x2,%xmm2,%xmm1 + aeskeygenassist $0x2,%xmm2,%xmm1 // round 4 call .Lkey_expansion_256a - aeskeygenassist $0x2,%xmm0,%xmm1 + aeskeygenassist $0x2,%xmm0,%xmm1 // round 5 call .Lkey_expansion_256b - aeskeygenassist $0x4,%xmm2,%xmm1 + aeskeygenassist $0x4,%xmm2,%xmm1 // round 6 call .Lkey_expansion_256a - aeskeygenassist $0x4,%xmm0,%xmm1 + aeskeygenassist $0x4,%xmm0,%xmm1 // round 7 call .Lkey_expansion_256b - aeskeygenassist $0x8,%xmm2,%xmm1 + aeskeygenassist $0x8,%xmm2,%xmm1 // round 8 call .Lkey_expansion_256a - aeskeygenassist $0x8,%xmm0,%xmm1 + aeskeygenassist $0x8,%xmm0,%xmm1 // round 9 call .Lkey_expansion_256b - aeskeygenassist $0x10,%xmm2,%xmm1 + aeskeygenassist $0x10,%xmm2,%xmm1 // round 10 call .Lkey_expansion_256a - aeskeygenassist $0x10,%xmm0,%xmm1 + aeskeygenassist $0x10,%xmm0,%xmm1 // round 11 call .Lkey_expansion_256b - aeskeygenassist $0x20,%xmm2,%xmm1 + aeskeygenassist $0x20,%xmm2,%xmm1 // round 12 call .Lkey_expansion_256a - aeskeygenassist $0x20,%xmm0,%xmm1 + aeskeygenassist $0x20,%xmm0,%xmm1 // round 13 call .Lkey_expansion_256b - aeskeygenassist $0x40,%xmm2,%xmm1 + aeskeygenassist $0x40,%xmm2,%xmm1 // round 14 call .Lkey_expansion_256a movups %xmm0,(%rax) - movl %esi,16(%rax) + movl %esi,16(%rax) // 240(%rdx) xorq %rax,%rax jmp .Lenc_key_ret @@ -2092,7 +2092,7 @@ xorps %xmm4,%xmm0 shufps $140,%xmm0,%xmm4 xorps %xmm4,%xmm0 - shufps $255,%xmm1,%xmm1 + shufps $255,%xmm1,%xmm1 // critical path xorps %xmm1,%xmm0 ret .cfi_endproc @@ -2111,7 +2111,7 @@ shufps $140,%xmm0,%xmm4 pslldq $4,%xmm3 xorps %xmm4,%xmm0 - pshufd $85,%xmm1,%xmm1 + pshufd $85,%xmm1,%xmm1 // critical path pxor %xmm3,%xmm2 pxor %xmm1,%xmm0 pshufd $255,%xmm0,%xmm3 @@ -2141,7 +2141,7 @@ xorps %xmm4,%xmm0 shufps $140,%xmm0,%xmm4 xorps %xmm4,%xmm0 - shufps $255,%xmm1,%xmm1 + shufps $255,%xmm1,%xmm1 // critical path xorps %xmm1,%xmm0 ret .cfi_endproc @@ -2156,7 +2156,7 @@ xorps %xmm4,%xmm2 shufps $140,%xmm2,%xmm4 xorps %xmm4,%xmm2 - shufps $170,%xmm1,%xmm1 + shufps $170,%xmm1,%xmm1 // critical path xorps %xmm1,%xmm2 ret .cfi_endproc @@ -2176,9 +2176,9 @@ .cfi_adjust_cfa_offset 8 - movups (%rdi),%xmm0 - xorps %xmm4,%xmm4 - leaq 16(%rdx),%rax + movups (%rdi),%xmm0 // pull first 128 bits of *userKey + xorps %xmm4,%xmm4 // low dword of xmm4 is assumed 0 + leaq 16(%rdx),%rax // %rax is used as modifiable copy of %rdx cmpl $256,%esi je .L14rounds_alt cmpl $192,%esi @@ -2186,7 +2186,7 @@ cmpl $128,%esi jne .Lbad_keybits_alt - movl $9,%esi + movl $9,%esi // 10 rounds for 128-bit key movdqa .Lkey_rotate(%rip),%xmm5 movl $8,%r10d movdqa .Lkey_rcon1(%rip),%xmm4 @@ -2248,14 +2248,14 @@ pxor %xmm2,%xmm0 movdqu %xmm0,16(%rax) - movl %esi,96(%rax) + movl %esi,96(%rax) // 240(%rdx) xorl %eax,%eax jmp .Lenc_key_ret_alt .align 16 .L12rounds_alt: - movq 16(%rdi),%xmm2 - movl $11,%esi + movq 16(%rdi),%xmm2 // remaining 1/3 of *userKey + movl $11,%esi // 12 rounds for 192 movdqa .Lkey_rotate192(%rip),%xmm5 movdqa .Lkey_rcon1(%rip),%xmm4 movl $8,%r10d @@ -2291,14 +2291,14 @@ decl %r10d jnz .Loop_key192 - movl %esi,32(%rax) + movl %esi,32(%rax) // 240(%rdx) xorl %eax,%eax jmp .Lenc_key_ret_alt .align 16 .L14rounds_alt: - movups 16(%rdi),%xmm2 - movl $13,%esi + movups 16(%rdi),%xmm2 // remaining half of *userKey + movl $13,%esi // 14 rounds for 256 leaq 16(%rax),%rax movdqa .Lkey_rotate(%rip),%xmm5 movdqa .Lkey_rcon1(%rip),%xmm4 @@ -2348,7 +2348,7 @@ jmp .Loop_key256 .Ldone_key256: - movl %esi,16(%rax) + movl %esi,16(%rax) // 240(%rdx) xorl %eax,%eax jmp .Lenc_key_ret_alt
diff --git a/gen/bcm/aesni-x86_64-win.asm b/gen/bcm/aesni-x86_64-win.asm index 18daf9a..7f55ea4 100644 --- a/gen/bcm/aesni-x86_64-win.asm +++ b/gen/bcm/aesni-x86_64-win.asm
@@ -23,8 +23,8 @@ EXTERN BORINGSSL_function_hit mov BYTE[((BORINGSSL_function_hit+1))],1 %endif - movups xmm2,XMMWORD[rcx] - mov eax,DWORD[240+r8] + movups xmm2,XMMWORD[rcx] ; load input + mov eax,DWORD[240+r8] ; key->rounds movups xmm0,XMMWORD[r8] movups xmm1,XMMWORD[16+r8] lea r8,[32+r8] @@ -34,11 +34,11 @@ dec eax movups xmm1,XMMWORD[r8] lea r8,[16+r8] - jnz NEAR $L$oop_enc1_1 + jnz NEAR $L$oop_enc1_1 ; loop body is 16 bytes aesenclast xmm2,xmm1 - pxor xmm0,xmm0 + pxor xmm0,xmm0 ; clear register bank pxor xmm1,xmm1 - movups XMMWORD[rdx],xmm2 + movups XMMWORD[rdx],xmm2 ; output pxor xmm2,xmm2 ret @@ -50,8 +50,8 @@ aes_hw_decrypt: _CET_ENDBR - movups xmm2,XMMWORD[rcx] - mov eax,DWORD[240+r8] + movups xmm2,XMMWORD[rcx] ; load input + mov eax,DWORD[240+r8] ; key->rounds movups xmm0,XMMWORD[r8] movups xmm1,XMMWORD[16+r8] lea r8,[32+r8] @@ -61,11 +61,11 @@ dec eax movups xmm1,XMMWORD[r8] lea r8,[16+r8] - jnz NEAR $L$oop_dec1_2 + jnz NEAR $L$oop_dec1_2 ; loop body is 16 bytes aesdeclast xmm2,xmm1 - pxor xmm0,xmm0 + pxor xmm0,xmm0 ; clear register bank pxor xmm1,xmm1 - movups XMMWORD[rdx],xmm2 + movups XMMWORD[rdx],xmm2 ; output pxor xmm2,xmm2 ret @@ -81,7 +81,7 @@ xorps xmm3,xmm0 movups xmm0,XMMWORD[32+rcx] lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax add rax,16 $L$enc_loop2: @@ -112,7 +112,7 @@ xorps xmm3,xmm0 movups xmm0,XMMWORD[32+rcx] lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax add rax,16 $L$dec_loop2: @@ -144,7 +144,7 @@ xorps xmm4,xmm0 movups xmm0,XMMWORD[32+rcx] lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax add rax,16 $L$enc_loop3: @@ -180,7 +180,7 @@ xorps xmm4,xmm0 movups xmm0,XMMWORD[32+rcx] lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax add rax,16 $L$dec_loop3: @@ -217,7 +217,7 @@ xorps xmm5,xmm0 movups xmm0,XMMWORD[32+rcx] lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax DB 0x0f,0x1f,0x00 add rax,16 @@ -259,7 +259,7 @@ xorps xmm5,xmm0 movups xmm0,XMMWORD[32+rcx] lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax DB 0x0f,0x1f,0x00 add rax,16 @@ -300,7 +300,7 @@ pxor xmm4,xmm0 aesenc xmm2,xmm1 lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax aesenc xmm3,xmm1 pxor xmm5,xmm0 pxor xmm6,xmm0 @@ -356,7 +356,7 @@ pxor xmm4,xmm0 aesdec xmm2,xmm1 lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax aesdec xmm3,xmm1 pxor xmm5,xmm0 pxor xmm6,xmm0 @@ -413,7 +413,7 @@ pxor xmm5,xmm0 pxor xmm6,xmm0 lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax aesenc xmm2,xmm1 pxor xmm7,xmm0 pxor xmm8,xmm0 @@ -479,7 +479,7 @@ pxor xmm5,xmm0 pxor xmm6,xmm0 lea rcx,[32+rax*1+rcx] - neg rax + neg rax ; %eax aesdec xmm2,xmm1 pxor xmm7,xmm0 pxor xmm8,xmm0 @@ -550,25 +550,25 @@ _CET_ENDBR lea rsp,[((-88))+rsp] - movaps XMMWORD[rsp],xmm6 + movaps XMMWORD[rsp],xmm6 ; offload %xmm6..7 movaps XMMWORD[16+rsp],xmm7 movaps XMMWORD[32+rsp],xmm8 movaps XMMWORD[48+rsp],xmm9 $L$ecb_enc_body: - and rdx,-16 - jz NEAR $L$ecb_ret + and rdx,-16 ; if (%rdx<16) + jz NEAR $L$ecb_ret ; return - mov eax,DWORD[240+rcx] + mov eax,DWORD[240+rcx] ; key->rounds movups xmm0,XMMWORD[rcx] - mov r11,rcx - mov r10d,eax - test r8d,r8d + mov r11,rcx ; backup %rcx + mov r10d,eax ; backup %eax + test r8d,r8d ; 5th argument jz NEAR $L$ecb_decrypt +; --------------------------- ECB ENCRYPT ------------------------------# + cmp rdx,0x80 ; if (%rdx<8*16) + jb NEAR $L$ecb_enc_tail ; short input - cmp rdx,0x80 - jb NEAR $L$ecb_enc_tail - - movdqu xmm2,XMMWORD[rdi] + movdqu xmm2,XMMWORD[rdi] ; load 8 input blocks movdqu xmm3,XMMWORD[16+rdi] movdqu xmm4,XMMWORD[32+rdi] movdqu xmm5,XMMWORD[48+rdi] @@ -576,15 +576,15 @@ movdqu xmm7,XMMWORD[80+rdi] movdqu xmm8,XMMWORD[96+rdi] movdqu xmm9,XMMWORD[112+rdi] - lea rdi,[128+rdi] - sub rdx,0x80 + lea rdi,[128+rdi] ; %rdi+=8*16 + sub rdx,0x80 ; %rdx-=8*16 (can be zero) jmp NEAR $L$ecb_enc_loop8_enter ALIGN 16 $L$ecb_enc_loop8: - movups XMMWORD[rsi],xmm2 - mov rcx,r11 - movdqu xmm2,XMMWORD[rdi] - mov eax,r10d + movups XMMWORD[rsi],xmm2 ; store 8 output blocks + mov rcx,r11 ; restore %rcx + movdqu xmm2,XMMWORD[rdi] ; load 8 input blocks + mov eax,r10d ; restore %eax movups XMMWORD[16+rsi],xmm3 movdqu xmm3,XMMWORD[16+rdi] movups XMMWORD[32+rsi],xmm4 @@ -598,31 +598,31 @@ movups XMMWORD[96+rsi],xmm8 movdqu xmm8,XMMWORD[96+rdi] movups XMMWORD[112+rsi],xmm9 - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; %rsi+=8*16 movdqu xmm9,XMMWORD[112+rdi] - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; %rdi+=8*16 $L$ecb_enc_loop8_enter: call _aesni_encrypt8 sub rdx,0x80 - jnc NEAR $L$ecb_enc_loop8 + jnc NEAR $L$ecb_enc_loop8 ; loop if %rdx-=8*16 didn't borrow - movups XMMWORD[rsi],xmm2 - mov rcx,r11 + movups XMMWORD[rsi],xmm2 ; store 8 output blocks + mov rcx,r11 ; restore %rcx movups XMMWORD[16+rsi],xmm3 - mov eax,r10d + mov eax,r10d ; restore %eax movups XMMWORD[32+rsi],xmm4 movups XMMWORD[48+rsi],xmm5 movups XMMWORD[64+rsi],xmm6 movups XMMWORD[80+rsi],xmm7 movups XMMWORD[96+rsi],xmm8 movups XMMWORD[112+rsi],xmm9 - lea rsi,[128+rsi] - add rdx,0x80 - jz NEAR $L$ecb_ret + lea rsi,[128+rsi] ; %rsi+=8*16 + add rdx,0x80 ; restore real remaining %rdx + jz NEAR $L$ecb_ret ; done if (%rdx==0) -$L$ecb_enc_tail: +$L$ecb_enc_tail: ; %rdx is less than 8*16 movups xmm2,XMMWORD[rdi] cmp rdx,0x20 jb NEAR $L$ecb_enc_one @@ -641,7 +641,7 @@ movdqu xmm8,XMMWORD[96+rdi] xorps xmm9,xmm9 call _aesni_encrypt8 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 7 output blocks movups XMMWORD[16+rsi],xmm3 movups XMMWORD[32+rsi],xmm4 movups XMMWORD[48+rsi],xmm5 @@ -660,27 +660,27 @@ dec eax movups xmm1,XMMWORD[rcx] lea rcx,[16+rcx] - jnz NEAR $L$oop_enc1_3 + jnz NEAR $L$oop_enc1_3 ; loop body is 16 bytes aesenclast xmm2,xmm1 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store one output block jmp NEAR $L$ecb_ret ALIGN 16 $L$ecb_enc_two: call _aesni_encrypt2 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 2 output blocks movups XMMWORD[16+rsi],xmm3 jmp NEAR $L$ecb_ret ALIGN 16 $L$ecb_enc_three: call _aesni_encrypt3 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 3 output blocks movups XMMWORD[16+rsi],xmm3 movups XMMWORD[32+rsi],xmm4 jmp NEAR $L$ecb_ret ALIGN 16 $L$ecb_enc_four: call _aesni_encrypt4 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 4 output blocks movups XMMWORD[16+rsi],xmm3 movups XMMWORD[32+rsi],xmm4 movups XMMWORD[48+rsi],xmm5 @@ -689,7 +689,7 @@ $L$ecb_enc_five: xorps xmm7,xmm7 call _aesni_encrypt6 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 5 output blocks movups XMMWORD[16+rsi],xmm3 movups XMMWORD[32+rsi],xmm4 movups XMMWORD[48+rsi],xmm5 @@ -698,20 +698,20 @@ ALIGN 16 $L$ecb_enc_six: call _aesni_encrypt6 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 6 output blocks movups XMMWORD[16+rsi],xmm3 movups XMMWORD[32+rsi],xmm4 movups XMMWORD[48+rsi],xmm5 movups XMMWORD[64+rsi],xmm6 movups XMMWORD[80+rsi],xmm7 jmp NEAR $L$ecb_ret - +; --------------------------- ECB DECRYPT ------------------------------# ALIGN 16 $L$ecb_decrypt: - cmp rdx,0x80 - jb NEAR $L$ecb_dec_tail + cmp rdx,0x80 ; if (%rdx<8*16) + jb NEAR $L$ecb_dec_tail ; short input - movdqu xmm2,XMMWORD[rdi] + movdqu xmm2,XMMWORD[rdi] ; load 8 input blocks movdqu xmm3,XMMWORD[16+rdi] movdqu xmm4,XMMWORD[32+rdi] movdqu xmm5,XMMWORD[48+rdi] @@ -719,15 +719,15 @@ movdqu xmm7,XMMWORD[80+rdi] movdqu xmm8,XMMWORD[96+rdi] movdqu xmm9,XMMWORD[112+rdi] - lea rdi,[128+rdi] - sub rdx,0x80 + lea rdi,[128+rdi] ; %rdi+=8*16 + sub rdx,0x80 ; %rdx-=8*16 (can be zero) jmp NEAR $L$ecb_dec_loop8_enter ALIGN 16 $L$ecb_dec_loop8: - movups XMMWORD[rsi],xmm2 - mov rcx,r11 - movdqu xmm2,XMMWORD[rdi] - mov eax,r10d + movups XMMWORD[rsi],xmm2 ; store 8 output blocks + mov rcx,r11 ; restore %rcx + movdqu xmm2,XMMWORD[rdi] ; load 8 input blocks + mov eax,r10d ; restore %eax movups XMMWORD[16+rsi],xmm3 movdqu xmm3,XMMWORD[16+rdi] movups XMMWORD[32+rsi],xmm4 @@ -741,23 +741,23 @@ movups XMMWORD[96+rsi],xmm8 movdqu xmm8,XMMWORD[96+rdi] movups XMMWORD[112+rsi],xmm9 - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; %rsi+=8*16 movdqu xmm9,XMMWORD[112+rdi] - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; %rdi+=8*16 $L$ecb_dec_loop8_enter: call _aesni_decrypt8 movups xmm0,XMMWORD[r11] sub rdx,0x80 - jnc NEAR $L$ecb_dec_loop8 + jnc NEAR $L$ecb_dec_loop8 ; loop if %rdx-=8*16 didn't borrow - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 - mov rcx,r11 + movups XMMWORD[rsi],xmm2 ; store 8 output blocks + pxor xmm2,xmm2 ; clear register bank + mov rcx,r11 ; restore %rcx movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 - mov eax,r10d + mov eax,r10d ; restore %eax movups XMMWORD[32+rsi],xmm4 pxor xmm4,xmm4 movups XMMWORD[48+rsi],xmm5 @@ -770,9 +770,9 @@ pxor xmm8,xmm8 movups XMMWORD[112+rsi],xmm9 pxor xmm9,xmm9 - lea rsi,[128+rsi] - add rdx,0x80 - jz NEAR $L$ecb_ret + lea rsi,[128+rsi] ; %rsi+=8*16 + add rdx,0x80 ; restore real remaining %rdx + jz NEAR $L$ecb_ret ; done if (%rdx==0) $L$ecb_dec_tail: movups xmm2,XMMWORD[rdi] @@ -794,8 +794,8 @@ movups xmm0,XMMWORD[rcx] xorps xmm9,xmm9 call _aesni_decrypt8 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store 7 output blocks + pxor xmm2,xmm2 ; clear register bank movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 movups XMMWORD[32+rsi],xmm4 @@ -821,24 +821,24 @@ dec eax movups xmm1,XMMWORD[rcx] lea rcx,[16+rcx] - jnz NEAR $L$oop_dec1_4 + jnz NEAR $L$oop_dec1_4 ; loop body is 16 bytes aesdeclast xmm2,xmm1 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store one output block + pxor xmm2,xmm2 ; clear register bank jmp NEAR $L$ecb_ret ALIGN 16 $L$ecb_dec_two: call _aesni_decrypt2 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store 2 output blocks + pxor xmm2,xmm2 ; clear register bank movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 jmp NEAR $L$ecb_ret ALIGN 16 $L$ecb_dec_three: call _aesni_decrypt3 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store 3 output blocks + pxor xmm2,xmm2 ; clear register bank movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 movups XMMWORD[32+rsi],xmm4 @@ -847,8 +847,8 @@ ALIGN 16 $L$ecb_dec_four: call _aesni_decrypt4 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store 4 output blocks + pxor xmm2,xmm2 ; clear register bank movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 movups XMMWORD[32+rsi],xmm4 @@ -860,8 +860,8 @@ $L$ecb_dec_five: xorps xmm7,xmm7 call _aesni_decrypt6 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store 5 output blocks + pxor xmm2,xmm2 ; clear register bank movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 movups XMMWORD[32+rsi],xmm4 @@ -875,8 +875,8 @@ ALIGN 16 $L$ecb_dec_six: call _aesni_decrypt6 - movups XMMWORD[rsi],xmm2 - pxor xmm2,xmm2 + movups XMMWORD[rsi],xmm2 ; store 6 output blocks + pxor xmm2,xmm2 ; clear register bank movups XMMWORD[16+rsi],xmm3 pxor xmm3,xmm3 movups XMMWORD[32+rsi],xmm4 @@ -889,10 +889,10 @@ pxor xmm7,xmm7 $L$ecb_ret: - xorps xmm0,xmm0 + xorps xmm0,xmm0 ; %xmm0 pxor xmm1,xmm1 movaps xmm6,XMMWORD[rsp] - movaps XMMWORD[rsp],xmm0 + movaps XMMWORD[rsp],xmm0 ; clear stack movaps xmm7,XMMWORD[16+rsp] movaps XMMWORD[16+rsp],xmm0 movaps xmm8,XMMWORD[32+rsp] @@ -929,11 +929,11 @@ cmp rdx,1 jne NEAR $L$ctr32_bulk - - +; handle single block without allocating stack frame, +; useful when handling edges movups xmm2,XMMWORD[r8] movups xmm3,XMMWORD[rdi] - mov edx,DWORD[240+rcx] + mov edx,DWORD[240+rcx] ; key->rounds movups xmm0,XMMWORD[rcx] movups xmm1,XMMWORD[16+rcx] lea rcx,[32+rcx] @@ -943,9 +943,9 @@ dec edx movups xmm1,XMMWORD[rcx] lea rcx,[16+rcx] - jnz NEAR $L$oop_enc1_5 + jnz NEAR $L$oop_enc1_5 ; loop body is 16 bytes aesenclast xmm2,xmm1 - pxor xmm0,xmm0 + pxor xmm0,xmm0 ; clear register bank pxor xmm1,xmm1 xorps xmm2,xmm3 pxor xmm3,xmm3 @@ -955,13 +955,13 @@ ALIGN 16 $L$ctr32_bulk: - lea r11,[rsp] + lea r11,[rsp] ; use %r11 as frame pointer push rbp sub rsp,288 - and rsp,-16 - movaps XMMWORD[(-168)+r11],xmm6 + and rsp,-16 ; Linux kernel stack can be incorrectly seeded + movaps XMMWORD[(-168)+r11],xmm6 ; offload everything movaps XMMWORD[(-152)+r11],xmm7 movaps XMMWORD[(-136)+r11],xmm8 movaps XMMWORD[(-120)+r11],xmm9 @@ -973,15 +973,15 @@ movaps XMMWORD[(-24)+r11],xmm15 $L$ctr32_body: - - +; 8 16-byte words on top of stack are counter values +; xor-ed with zero-round key movdqu xmm2,XMMWORD[r8] movdqu xmm0,XMMWORD[rcx] - mov r8d,DWORD[12+r8] + mov r8d,DWORD[12+r8] ; counter LSB pxor xmm2,xmm0 - mov ebp,DWORD[12+rcx] - movdqa XMMWORD[rsp],xmm2 + mov ebp,DWORD[12+rcx] ; 0-round key LSB + movdqa XMMWORD[rsp],xmm2 ; populate counter block bswap r8d movdqa xmm3,xmm2 movdqa xmm4,xmm2 @@ -989,7 +989,7 @@ movdqa XMMWORD[64+rsp],xmm2 movdqa XMMWORD[80+rsp],xmm2 movdqa XMMWORD[96+rsp],xmm2 - mov r10,rdx + mov r10,rdx ; about to borrow %rdx movdqa XMMWORD[112+rsp],xmm2 lea rax,[1+r8] @@ -1003,7 +1003,7 @@ movdqa XMMWORD[16+rsp],xmm3 pinsrd xmm4,edx,3 bswap eax - mov rdx,r10 + mov rdx,r10 ; restore %rdx lea r10,[4+r8] movdqa XMMWORD[32+rsp],xmm4 xor eax,ebp @@ -1015,7 +1015,7 @@ mov DWORD[((64+12))+rsp],r10d bswap r9d lea r10,[6+r8] - mov eax,DWORD[240+rcx] + mov eax,DWORD[240+rcx] ; key->rounds xor r9d,ebp bswap r10d mov DWORD[((80+12))+rsp],r9d @@ -1031,16 +1031,16 @@ movdqa xmm6,XMMWORD[64+rsp] movdqa xmm7,XMMWORD[80+rsp] - cmp rdx,8 - jb NEAR $L$ctr32_tail + cmp rdx,8 ; %rdx is in blocks + jb NEAR $L$ctr32_tail ; short input if (%rdx<8) - lea rcx,[128+rcx] - sub rdx,8 + lea rcx,[128+rcx] ; size optimization + sub rdx,8 ; %rdx is biased by -8 jmp NEAR $L$ctr32_loop8 ALIGN 32 $L$ctr32_loop8: - add r8d,8 + add r8d,8 ; next counter value movdqa xmm8,XMMWORD[96+rsp] aesenc xmm2,xmm1 mov r9d,r8d @@ -1052,7 +1052,7 @@ xor r9d,ebp nop aesenc xmm5,xmm1 - mov DWORD[((0+12))+rsp],r9d + mov DWORD[((0+12))+rsp],r9d ; store next counter value lea r9,[1+r8] aesenc xmm6,xmm1 aesenc xmm7,xmm1 @@ -1148,7 +1148,7 @@ aesenc xmm3,xmm0 aesenc xmm4,xmm0 xor r9d,ebp - movdqu xmm10,XMMWORD[rdi] + movdqu xmm10,XMMWORD[rdi] ; start loading input aesenc xmm5,xmm0 mov DWORD[((112+12))+rsp],r9d cmp eax,11 @@ -1205,7 +1205,7 @@ ALIGN 16 $L$ctr32_enc_done: movdqu xmm11,XMMWORD[16+rdi] - pxor xmm10,xmm0 + pxor xmm10,xmm0 ; input^=round[last] movdqu xmm12,XMMWORD[32+rdi] pxor xmm11,xmm0 movdqu xmm13,XMMWORD[48+rdi] @@ -1214,8 +1214,8 @@ pxor xmm13,xmm0 movdqu xmm15,XMMWORD[80+rdi] pxor xmm14,xmm0 - prefetcht0 [448+rdi] - prefetcht0 [512+rdi] + prefetcht0 [448+rdi] ; We process 128 bytes (8*16), so to prefetch 1 iteration + prefetcht0 [512+rdi] ; We need to prefetch 2 64 byte lines pxor xmm15,xmm0 aesenc xmm2,xmm1 aesenc xmm3,xmm1 @@ -1225,15 +1225,15 @@ aesenc xmm7,xmm1 aesenc xmm8,xmm1 aesenc xmm9,xmm1 - movdqu xmm1,XMMWORD[96+rdi] - lea rdi,[128+rdi] + movdqu xmm1,XMMWORD[96+rdi] ; borrow %xmm1 for inp[6] + lea rdi,[128+rdi] ; %rdi+=8*16 - aesenclast xmm2,xmm10 - pxor xmm1,xmm0 + aesenclast xmm2,xmm10 ; is inp[N]^round[last] + pxor xmm1,xmm0 ; borrowed movdqu xmm10,XMMWORD[((112-128))+rdi] aesenclast xmm3,xmm11 pxor xmm10,xmm0 - movdqa xmm11,XMMWORD[rsp] + movdqa xmm11,XMMWORD[rsp] ; load next counter block aesenclast xmm4,xmm12 aesenclast xmm5,xmm13 movdqa xmm12,XMMWORD[16+rsp] @@ -1244,10 +1244,10 @@ movdqa xmm15,XMMWORD[64+rsp] aesenclast xmm8,xmm1 movdqa xmm0,XMMWORD[80+rsp] - movups xmm1,XMMWORD[((16-128))+rcx] + movups xmm1,XMMWORD[((16-128))+rcx] ; real 1st-round key aesenclast xmm9,xmm10 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store 8 output blocks movdqa xmm2,xmm11 movups XMMWORD[16+rsi],xmm3 movdqa xmm3,xmm12 @@ -1261,24 +1261,24 @@ movdqa xmm7,xmm0 movups XMMWORD[96+rsi],xmm8 movups XMMWORD[112+rsi],xmm9 - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; %rsi+=8*16 sub rdx,8 - jnc NEAR $L$ctr32_loop8 + jnc NEAR $L$ctr32_loop8 ; loop if %rdx-=8 didn't borrow - add rdx,8 - jz NEAR $L$ctr32_done + add rdx,8 ; restore real remaining %rdx + jz NEAR $L$ctr32_done ; done if (%rdx==0) lea rcx,[((-128))+rcx] $L$ctr32_tail: - - +; note that at this point %xmm2..5 are populated with +; counter values xor-ed with 0-round key lea rcx,[16+rcx] cmp rdx,4 jb NEAR $L$ctr32_loop3 je NEAR $L$ctr32_loop4 - +; if (%rdx>4) compute 7 E(counter) shl eax,4 movdqa xmm8,XMMWORD[96+rsp] pxor xmm9,xmm9 @@ -1286,14 +1286,14 @@ movups xmm0,XMMWORD[16+rcx] aesenc xmm2,xmm1 aesenc xmm3,xmm1 - lea rcx,[((32-16))+rax*1+rcx] + lea rcx,[((32-16))+rax*1+rcx] ; prepare for .Lenc_loop8_enter neg rax aesenc xmm4,xmm1 - add rax,16 + add rax,16 ; prepare for .Lenc_loop8_enter movups xmm10,XMMWORD[rdi] aesenc xmm5,xmm1 aesenc xmm6,xmm1 - movups xmm11,XMMWORD[16+rdi] + movups xmm11,XMMWORD[16+rdi] ; pre-load input movups xmm12,XMMWORD[32+rdi] aesenc xmm7,xmm1 aesenc xmm8,xmm1 @@ -1304,7 +1304,7 @@ pxor xmm2,xmm10 movdqu xmm10,XMMWORD[64+rdi] pxor xmm3,xmm11 - movdqu XMMWORD[rsi],xmm2 + movdqu XMMWORD[rsi],xmm2 ; store output pxor xmm4,xmm12 movdqu XMMWORD[16+rsi],xmm3 pxor xmm5,xmm13 @@ -1313,17 +1313,17 @@ movdqu XMMWORD[48+rsi],xmm5 movdqu XMMWORD[64+rsi],xmm6 cmp rdx,6 - jb NEAR $L$ctr32_done + jb NEAR $L$ctr32_done ; %rdx was 5, stop store movups xmm11,XMMWORD[80+rdi] xorps xmm7,xmm11 movups XMMWORD[80+rsi],xmm7 - je NEAR $L$ctr32_done + je NEAR $L$ctr32_done ; %rdx was 6, stop store movups xmm12,XMMWORD[96+rdi] xorps xmm8,xmm12 movups XMMWORD[96+rsi],xmm8 - jmp NEAR $L$ctr32_done + jmp NEAR $L$ctr32_done ; %rdx was 7, stop store ALIGN 32 $L$ctr32_loop4: @@ -1337,7 +1337,7 @@ jnz NEAR $L$ctr32_loop4 aesenclast xmm2,xmm1 aesenclast xmm3,xmm1 - movups xmm10,XMMWORD[rdi] + movups xmm10,XMMWORD[rdi] ; load input movups xmm11,XMMWORD[16+rdi] aesenclast xmm4,xmm1 aesenclast xmm5,xmm1 @@ -1345,14 +1345,14 @@ movups xmm13,XMMWORD[48+rdi] xorps xmm2,xmm10 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store output xorps xmm3,xmm11 movups XMMWORD[16+rsi],xmm3 pxor xmm4,xmm12 movdqu XMMWORD[32+rsi],xmm4 pxor xmm5,xmm13 movdqu XMMWORD[48+rsi],xmm5 - jmp NEAR $L$ctr32_done + jmp NEAR $L$ctr32_done ; %rdx was 4, stop store ALIGN 32 $L$ctr32_loop3: @@ -1367,23 +1367,23 @@ aesenclast xmm3,xmm1 aesenclast xmm4,xmm1 - movups xmm10,XMMWORD[rdi] + movups xmm10,XMMWORD[rdi] ; load input xorps xmm2,xmm10 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store output cmp rdx,2 - jb NEAR $L$ctr32_done + jb NEAR $L$ctr32_done ; %rdx was 1, stop store movups xmm11,XMMWORD[16+rdi] xorps xmm3,xmm11 movups XMMWORD[16+rsi],xmm3 - je NEAR $L$ctr32_done + je NEAR $L$ctr32_done ; %rdx was 2, stop store movups xmm12,XMMWORD[32+rdi] xorps xmm4,xmm12 - movups XMMWORD[32+rsi],xmm4 + movups XMMWORD[32+rsi],xmm4 ; %rdx was 3, stop store $L$ctr32_done: - xorps xmm0,xmm0 + xorps xmm0,xmm0 ; clear register bank xor ebp,ebp pxor xmm1,xmm1 pxor xmm2,xmm2 @@ -1391,7 +1391,7 @@ pxor xmm4,xmm4 pxor xmm5,xmm5 movaps xmm6,XMMWORD[((-168))+r11] - movaps XMMWORD[(-168)+r11],xmm0 + movaps XMMWORD[(-168)+r11],xmm0 ; clear stack movaps xmm7,XMMWORD[((-152))+r11] movaps XMMWORD[(-152)+r11],xmm0 movaps xmm8,XMMWORD[((-136))+r11] @@ -1446,15 +1446,15 @@ _CET_ENDBR - test rdx,rdx + test rdx,rdx ; check length jz NEAR $L$cbc_ret - mov r10d,DWORD[240+rcx] - mov r11,rcx - test r9d,r9d + mov r10d,DWORD[240+rcx] ; key->rounds + mov r11,rcx ; backup %rcx + test r9d,r9d ; 6th argument jz NEAR $L$cbc_decrypt - - movups xmm2,XMMWORD[r8] +; --------------------------- CBC ENCRYPT ------------------------------# + movups xmm2,XMMWORD[r8] ; load iv as initial state mov eax,r10d cmp rdx,16 jb NEAR $L$cbc_enc_tail @@ -1462,9 +1462,9 @@ jmp NEAR $L$cbc_enc_loop ALIGN 16 $L$cbc_enc_loop: - movups xmm3,XMMWORD[rdi] + movups xmm3,XMMWORD[rdi] ; load input lea rdi,[16+rdi] - +; xorps %xmm3,%xmm2 movups xmm0,XMMWORD[rcx] movups xmm1,XMMWORD[16+rcx] xorps xmm3,xmm0 @@ -1475,17 +1475,17 @@ dec eax movups xmm1,XMMWORD[rcx] lea rcx,[16+rcx] - jnz NEAR $L$oop_enc1_6 + jnz NEAR $L$oop_enc1_6 ; loop body is 16 bytes aesenclast xmm2,xmm1 - mov eax,r10d - mov rcx,r11 - movups XMMWORD[rsi],xmm2 + mov eax,r10d ; restore %eax + mov rcx,r11 ; restore %rcx + movups XMMWORD[rsi],xmm2 ; store output lea rsi,[16+rsi] sub rdx,16 jnc NEAR $L$cbc_enc_loop add rdx,16 jnz NEAR $L$cbc_enc_tail - pxor xmm0,xmm0 + pxor xmm0,xmm0 ; clear register bank pxor xmm1,xmm1 movups XMMWORD[r8],xmm2 pxor xmm2,xmm2 @@ -1493,30 +1493,30 @@ jmp NEAR $L$cbc_ret $L$cbc_enc_tail: - mov rcx,rdx - xchg rsi,rdi - DD 0x9066A4F3 - mov ecx,16 + mov rcx,rdx ; zaps %rcx + xchg rsi,rdi ; %rdi is %rsi and %rsi is %rdi now + DD 0x9066A4F3 ; rep movsb + mov ecx,16 ; zero tail sub rcx,rdx xor eax,eax - DD 0x9066AAF3 - lea rdi,[((-16))+rdi] - mov eax,r10d - mov rsi,rdi - mov rcx,r11 - xor rdx,rdx - jmp NEAR $L$cbc_enc_loop - + DD 0x9066AAF3 ; rep stosb + lea rdi,[((-16))+rdi] ; rewind %rsi by 1 block + mov eax,r10d ; restore %eax + mov rsi,rdi ; %rdi and %rsi are the same + mov rcx,r11 ; restore %rcx + xor rdx,rdx ; len=16 + jmp NEAR $L$cbc_enc_loop ; one more spin +; --------------------------- CBC DECRYPT ------------------------------# ALIGN 16 $L$cbc_decrypt: cmp rdx,16 jne NEAR $L$cbc_decrypt_bulk - - - movdqu xmm2,XMMWORD[rdi] - movdqu xmm3,XMMWORD[r8] - movdqa xmm4,xmm2 +; handle single block without allocating stack frame, +; useful in ciphertext stealing mode + movdqu xmm2,XMMWORD[rdi] ; load input + movdqu xmm3,XMMWORD[r8] ; load iv + movdqa xmm4,xmm2 ; future iv movups xmm0,XMMWORD[rcx] movups xmm1,XMMWORD[16+rcx] lea rcx,[32+rcx] @@ -1526,24 +1526,24 @@ dec r10d movups xmm1,XMMWORD[rcx] lea rcx,[16+rcx] - jnz NEAR $L$oop_dec1_7 + jnz NEAR $L$oop_dec1_7 ; loop body is 16 bytes aesdeclast xmm2,xmm1 - pxor xmm0,xmm0 + pxor xmm0,xmm0 ; clear register bank pxor xmm1,xmm1 - movdqu XMMWORD[r8],xmm4 - xorps xmm2,xmm3 + movdqu XMMWORD[r8],xmm4 ; store iv + xorps xmm2,xmm3 ; ^=iv pxor xmm3,xmm3 - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store output pxor xmm2,xmm2 jmp NEAR $L$cbc_ret ALIGN 16 $L$cbc_decrypt_bulk: - lea r11,[rsp] + lea r11,[rsp] ; frame pointer push rbp sub rsp,176 - and rsp,-16 + and rsp,-16 ; Linux kernel stack can be incorrectly seeded movaps XMMWORD[16+rsp],xmm6 movaps XMMWORD[32+rsp],xmm7 movaps XMMWORD[48+rsp],xmm8 @@ -1555,14 +1555,14 @@ movaps XMMWORD[144+rsp],xmm14 movaps XMMWORD[160+rsp],xmm15 $L$cbc_decrypt_body: - mov rbp,rcx + mov rbp,rcx ; [re-]backup %rcx [after reassignment] movups xmm10,XMMWORD[r8] mov eax,r10d cmp rdx,0x50 jbe NEAR $L$cbc_dec_tail movups xmm0,XMMWORD[rcx] - movdqu xmm2,XMMWORD[rdi] + movdqu xmm2,XMMWORD[rdi] ; load input movdqu xmm3,XMMWORD[16+rdi] movdqa xmm11,xmm2 movdqu xmm4,XMMWORD[32+rdi] @@ -1576,8 +1576,8 @@ cmp rdx,0x70 jbe NEAR $L$cbc_dec_six_or_seven - sub rdx,0x70 - lea rcx,[112+rcx] + sub rdx,0x70 ; %rdx is biased by -7*16 + lea rcx,[112+rcx] ; size optimization jmp NEAR $L$cbc_dec_loop8_enter ALIGN 16 $L$cbc_dec_loop8: @@ -1591,7 +1591,7 @@ movups xmm1,XMMWORD[((16-112))+rcx] pxor xmm4,xmm0 mov rbp,-1 - cmp rdx,0x70 + cmp rdx,0x70 ; is there at least 0x60 bytes ahead? pxor xmm5,xmm0 pxor xmm6,xmm0 pxor xmm7,xmm0 @@ -1740,11 +1740,11 @@ movdqu xmm1,XMMWORD[80+rdi] aesdeclast xmm2,xmm10 - movdqu xmm10,XMMWORD[96+rdi] + movdqu xmm10,XMMWORD[96+rdi] ; borrow %xmm10 pxor xmm1,xmm0 aesdeclast xmm3,xmm11 pxor xmm10,xmm0 - movdqu xmm0,XMMWORD[112+rdi] + movdqu xmm0,XMMWORD[112+rdi] ; next IV aesdeclast xmm4,xmm12 lea rdi,[128+rdi] movdqu xmm11,XMMWORD[rbp] @@ -1757,11 +1757,11 @@ movdqu xmm14,XMMWORD[48+rbp] movdqu xmm15,XMMWORD[64+rbp] aesdeclast xmm9,xmm10 - movdqa xmm10,xmm0 + movdqa xmm10,xmm0 ; return %xmm10 movdqu xmm1,XMMWORD[80+rbp] movups xmm0,XMMWORD[((-112))+rcx] - movups XMMWORD[rsi],xmm2 + movups XMMWORD[rsi],xmm2 ; store output movdqa xmm2,xmm11 movups XMMWORD[16+rsi],xmm3 movdqa xmm3,xmm12 @@ -1795,13 +1795,13 @@ movaps xmm8,xmm7 call _aesni_decrypt6 - pxor xmm2,xmm10 + pxor xmm2,xmm10 ; ^= IV movaps xmm10,xmm8 pxor xmm3,xmm11 movdqu XMMWORD[rsi],xmm2 pxor xmm4,xmm12 movdqu XMMWORD[16+rsi],xmm3 - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank pxor xmm5,xmm13 movdqu XMMWORD[32+rsi],xmm4 pxor xmm4,xmm4 @@ -1822,13 +1822,13 @@ xorps xmm9,xmm9 call _aesni_decrypt8 movups xmm9,XMMWORD[80+rdi] - pxor xmm2,xmm10 + pxor xmm2,xmm10 ; ^= IV movups xmm10,XMMWORD[96+rdi] pxor xmm3,xmm11 movdqu XMMWORD[rsi],xmm2 pxor xmm4,xmm12 movdqu XMMWORD[16+rsi],xmm3 - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank pxor xmm5,xmm13 movdqu XMMWORD[32+rsi],xmm4 pxor xmm4,xmm4 @@ -1850,24 +1850,24 @@ $L$cbc_dec_tail: movups xmm2,XMMWORD[rdi] sub rdx,0x10 - jbe NEAR $L$cbc_dec_one + jbe NEAR $L$cbc_dec_one ; %rdx is 1*16 or less movups xmm3,XMMWORD[16+rdi] movaps xmm11,xmm2 sub rdx,0x10 - jbe NEAR $L$cbc_dec_two + jbe NEAR $L$cbc_dec_two ; %rdx is 2*16 or less movups xmm4,XMMWORD[32+rdi] movaps xmm12,xmm3 sub rdx,0x10 - jbe NEAR $L$cbc_dec_three + jbe NEAR $L$cbc_dec_three ; %rdx is 3*16 or less movups xmm5,XMMWORD[48+rdi] movaps xmm13,xmm4 sub rdx,0x10 - jbe NEAR $L$cbc_dec_four + jbe NEAR $L$cbc_dec_four ; %rdx is 4*16 or less - movups xmm6,XMMWORD[64+rdi] + movups xmm6,XMMWORD[64+rdi] ; %rdx is 5*16 or less movaps xmm14,xmm5 movaps xmm15,xmm6 xorps xmm7,xmm7 @@ -1878,7 +1878,7 @@ movdqu XMMWORD[rsi],xmm2 pxor xmm4,xmm12 movdqu XMMWORD[16+rsi],xmm3 - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank pxor xmm5,xmm13 movdqu XMMWORD[32+rsi],xmm4 pxor xmm4,xmm4 @@ -1904,7 +1904,7 @@ dec eax movups xmm1,XMMWORD[rcx] lea rcx,[16+rcx] - jnz NEAR $L$oop_dec1_8 + jnz NEAR $L$oop_dec1_8 ; loop body is 16 bytes aesdeclast xmm2,xmm1 xorps xmm2,xmm10 movaps xmm10,xmm11 @@ -1918,7 +1918,7 @@ pxor xmm3,xmm11 movdqu XMMWORD[rsi],xmm2 movdqa xmm2,xmm3 - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank lea rsi,[16+rsi] jmp NEAR $L$cbc_dec_tail_collected ALIGN 16 @@ -1931,7 +1931,7 @@ movdqu XMMWORD[rsi],xmm2 pxor xmm4,xmm12 movdqu XMMWORD[16+rsi],xmm3 - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank movdqa xmm2,xmm4 pxor xmm4,xmm4 lea rsi,[32+rsi] @@ -1946,7 +1946,7 @@ movdqu XMMWORD[rsi],xmm2 pxor xmm4,xmm12 movdqu XMMWORD[16+rsi],xmm3 - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank pxor xmm5,xmm13 movdqu XMMWORD[32+rsi],xmm4 pxor xmm4,xmm4 @@ -1957,7 +1957,7 @@ ALIGN 16 $L$cbc_dec_clear_tail_collected: - pxor xmm3,xmm3 + pxor xmm3,xmm3 ; clear register bank pxor xmm4,xmm4 pxor xmm5,xmm5 $L$cbc_dec_tail_collected: @@ -1975,14 +1975,14 @@ mov rdi,rsi sub rcx,rdx lea rsi,[rsp] - DD 0x9066A4F3 + DD 0x9066A4F3 ; rep movsb movdqa XMMWORD[rsp],xmm2 $L$cbc_dec_ret: - xorps xmm0,xmm0 + xorps xmm0,xmm0 ; %xmm0 pxor xmm1,xmm1 movaps xmm6,XMMWORD[16+rsp] - movaps XMMWORD[16+rsp],xmm0 + movaps XMMWORD[16+rsp],xmm0 ; clear stack movaps xmm7,XMMWORD[32+rsp] movaps XMMWORD[32+rsp],xmm0 movaps xmm8,XMMWORD[48+rsp] @@ -2021,9 +2021,9 @@ mov edx,DWORD[240+rcx] shl edx,4 - lea r8,[16+rdx*1+rcx] + lea r8,[16+rdx*1+rcx] ; points at the end of key schedule - movups xmm0,XMMWORD[rcx] + movups xmm0,XMMWORD[rcx] ; just swap movups xmm1,XMMWORD[r8] movups XMMWORD[r8],xmm0 movups XMMWORD[rcx],xmm1 @@ -2031,7 +2031,7 @@ lea r8,[((-16))+r8] $L$dec_key_inverse: - movups xmm0,XMMWORD[rcx] + movups xmm0,XMMWORD[rcx] ; swap and inverse movups xmm1,XMMWORD[r8] aesimc xmm0,xmm0 aesimc xmm1,xmm1 @@ -2042,7 +2042,7 @@ cmp r8,rcx ja NEAR $L$dec_key_inverse - movups xmm0,XMMWORD[rcx] + movups xmm0,XMMWORD[rcx] ; inverse middle aesimc xmm0,xmm0 pxor xmm1,xmm1 movups XMMWORD[r8],xmm0 @@ -2064,9 +2064,9 @@ $L$SEH_prologue_aes_hw_set_encrypt_key_base_2: $L$SEH_endprologue_aes_hw_set_encrypt_key_base_3: - movups xmm0,XMMWORD[rcx] - xorps xmm4,xmm4 - lea rax,[16+r8] + movups xmm0,XMMWORD[rcx] ; pull first 128 bits of *userKey + xorps xmm4,xmm4 ; low dword of xmm4 is assumed 0 + lea rax,[16+r8] ; %rax is used as modifiable copy of %r8 cmp edx,256 je NEAR $L$14rounds cmp edx,192 @@ -2075,97 +2075,97 @@ jne NEAR $L$bad_keybits $L$10rounds: - mov edx,9 + mov edx,9 ; 10 rounds for 128-bit key - movups XMMWORD[r8],xmm0 - aeskeygenassist xmm1,xmm0,0x1 + movups XMMWORD[r8],xmm0 ; round 0 + aeskeygenassist xmm1,xmm0,0x1 ; round 1 call $L$key_expansion_128_cold - aeskeygenassist xmm1,xmm0,0x2 + aeskeygenassist xmm1,xmm0,0x2 ; round 2 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x4 + aeskeygenassist xmm1,xmm0,0x4 ; round 3 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x8 + aeskeygenassist xmm1,xmm0,0x8 ; round 4 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x10 + aeskeygenassist xmm1,xmm0,0x10 ; round 5 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x20 + aeskeygenassist xmm1,xmm0,0x20 ; round 6 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x40 + aeskeygenassist xmm1,xmm0,0x40 ; round 7 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x80 + aeskeygenassist xmm1,xmm0,0x80 ; round 8 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x1b + aeskeygenassist xmm1,xmm0,0x1b ; round 9 call $L$key_expansion_128 - aeskeygenassist xmm1,xmm0,0x36 + aeskeygenassist xmm1,xmm0,0x36 ; round 10 call $L$key_expansion_128 movups XMMWORD[rax],xmm0 - mov DWORD[80+rax],edx + mov DWORD[80+rax],edx ; 240(%rdx) xor eax,eax jmp NEAR $L$enc_key_ret ALIGN 16 $L$12rounds: - movq xmm2,QWORD[16+rcx] - mov edx,11 + movq xmm2,QWORD[16+rcx] ; remaining 1/3 of *userKey + mov edx,11 ; 12 rounds for 192 - movups XMMWORD[r8],xmm0 - aeskeygenassist xmm1,xmm2,0x1 + movups XMMWORD[r8],xmm0 ; round 0 + aeskeygenassist xmm1,xmm2,0x1 ; round 1,2 call $L$key_expansion_192a_cold - aeskeygenassist xmm1,xmm2,0x2 + aeskeygenassist xmm1,xmm2,0x2 ; round 2,3 call $L$key_expansion_192b - aeskeygenassist xmm1,xmm2,0x4 + aeskeygenassist xmm1,xmm2,0x4 ; round 4,5 call $L$key_expansion_192a - aeskeygenassist xmm1,xmm2,0x8 + aeskeygenassist xmm1,xmm2,0x8 ; round 5,6 call $L$key_expansion_192b - aeskeygenassist xmm1,xmm2,0x10 + aeskeygenassist xmm1,xmm2,0x10 ; round 7,8 call $L$key_expansion_192a - aeskeygenassist xmm1,xmm2,0x20 + aeskeygenassist xmm1,xmm2,0x20 ; round 8,9 call $L$key_expansion_192b - aeskeygenassist xmm1,xmm2,0x40 + aeskeygenassist xmm1,xmm2,0x40 ; round 10,11 call $L$key_expansion_192a - aeskeygenassist xmm1,xmm2,0x80 + aeskeygenassist xmm1,xmm2,0x80 ; round 11,12 call $L$key_expansion_192b movups XMMWORD[rax],xmm0 - mov DWORD[48+rax],edx + mov DWORD[48+rax],edx ; 240(%rdx) xor rax,rax jmp NEAR $L$enc_key_ret ALIGN 16 $L$14rounds: - movups xmm2,XMMWORD[16+rcx] - mov edx,13 + movups xmm2,XMMWORD[16+rcx] ; remaining half of *userKey + mov edx,13 ; 14 rounds for 256 lea rax,[16+rax] - movups XMMWORD[r8],xmm0 - movups XMMWORD[16+r8],xmm2 - aeskeygenassist xmm1,xmm2,0x1 + movups XMMWORD[r8],xmm0 ; round 0 + movups XMMWORD[16+r8],xmm2 ; round 1 + aeskeygenassist xmm1,xmm2,0x1 ; round 2 call $L$key_expansion_256a_cold - aeskeygenassist xmm1,xmm0,0x1 + aeskeygenassist xmm1,xmm0,0x1 ; round 3 call $L$key_expansion_256b - aeskeygenassist xmm1,xmm2,0x2 + aeskeygenassist xmm1,xmm2,0x2 ; round 4 call $L$key_expansion_256a - aeskeygenassist xmm1,xmm0,0x2 + aeskeygenassist xmm1,xmm0,0x2 ; round 5 call $L$key_expansion_256b - aeskeygenassist xmm1,xmm2,0x4 + aeskeygenassist xmm1,xmm2,0x4 ; round 6 call $L$key_expansion_256a - aeskeygenassist xmm1,xmm0,0x4 + aeskeygenassist xmm1,xmm0,0x4 ; round 7 call $L$key_expansion_256b - aeskeygenassist xmm1,xmm2,0x8 + aeskeygenassist xmm1,xmm2,0x8 ; round 8 call $L$key_expansion_256a - aeskeygenassist xmm1,xmm0,0x8 + aeskeygenassist xmm1,xmm0,0x8 ; round 9 call $L$key_expansion_256b - aeskeygenassist xmm1,xmm2,0x10 + aeskeygenassist xmm1,xmm2,0x10 ; round 10 call $L$key_expansion_256a - aeskeygenassist xmm1,xmm0,0x10 + aeskeygenassist xmm1,xmm0,0x10 ; round 11 call $L$key_expansion_256b - aeskeygenassist xmm1,xmm2,0x20 + aeskeygenassist xmm1,xmm2,0x20 ; round 12 call $L$key_expansion_256a - aeskeygenassist xmm1,xmm0,0x20 + aeskeygenassist xmm1,xmm0,0x20 ; round 13 call $L$key_expansion_256b - aeskeygenassist xmm1,xmm2,0x40 + aeskeygenassist xmm1,xmm2,0x40 ; round 14 call $L$key_expansion_256a movups XMMWORD[rax],xmm0 - mov DWORD[16+rax],edx + mov DWORD[16+rax],edx ; 240(%rdx) xor rax,rax jmp NEAR $L$enc_key_ret @@ -2196,7 +2196,7 @@ xorps xmm0,xmm4 shufps xmm4,xmm0,140 xorps xmm0,xmm4 - shufps xmm1,xmm1,255 + shufps xmm1,xmm1,255 ; critical path xorps xmm0,xmm1 ret @@ -2215,7 +2215,7 @@ shufps xmm4,xmm0,140 pslldq xmm3,4 xorps xmm0,xmm4 - pshufd xmm1,xmm1,85 + pshufd xmm1,xmm1,85 ; critical path pxor xmm2,xmm3 pxor xmm0,xmm1 pshufd xmm3,xmm0,255 @@ -2245,7 +2245,7 @@ xorps xmm0,xmm4 shufps xmm4,xmm0,140 xorps xmm0,xmm4 - shufps xmm1,xmm1,255 + shufps xmm1,xmm1,255 ; critical path xorps xmm0,xmm1 ret @@ -2260,7 +2260,7 @@ xorps xmm2,xmm4 shufps xmm4,xmm2,140 xorps xmm2,xmm4 - shufps xmm1,xmm1,170 + shufps xmm1,xmm1,170 ; critical path xorps xmm2,xmm1 ret @@ -2279,9 +2279,9 @@ $L$SEH_prologue_aes_hw_set_encrypt_key_alt_2: $L$SEH_endprologue_aes_hw_set_encrypt_key_alt_3: - movups xmm0,XMMWORD[rcx] - xorps xmm4,xmm4 - lea rax,[16+r8] + movups xmm0,XMMWORD[rcx] ; pull first 128 bits of *userKey + xorps xmm4,xmm4 ; low dword of xmm4 is assumed 0 + lea rax,[16+r8] ; %rax is used as modifiable copy of %r8 cmp edx,256 je NEAR $L$14rounds_alt cmp edx,192 @@ -2289,7 +2289,7 @@ cmp edx,128 jne NEAR $L$bad_keybits_alt - mov edx,9 + mov edx,9 ; 10 rounds for 128-bit key movdqa xmm5,XMMWORD[$L$key_rotate] mov r10d,8 movdqa xmm4,XMMWORD[$L$key_rcon1] @@ -2351,14 +2351,14 @@ pxor xmm0,xmm2 movdqu XMMWORD[16+rax],xmm0 - mov DWORD[96+rax],edx + mov DWORD[96+rax],edx ; 240(%r8) xor eax,eax jmp NEAR $L$enc_key_ret_alt ALIGN 16 $L$12rounds_alt: - movq xmm2,QWORD[16+rcx] - mov edx,11 + movq xmm2,QWORD[16+rcx] ; remaining 1/3 of *userKey + mov edx,11 ; 12 rounds for 192 movdqa xmm5,XMMWORD[$L$key_rotate192] movdqa xmm4,XMMWORD[$L$key_rcon1] mov r10d,8 @@ -2394,14 +2394,14 @@ dec r10d jnz NEAR $L$oop_key192 - mov DWORD[32+rax],edx + mov DWORD[32+rax],edx ; 240(%r8) xor eax,eax jmp NEAR $L$enc_key_ret_alt ALIGN 16 $L$14rounds_alt: - movups xmm2,XMMWORD[16+rcx] - mov edx,13 + movups xmm2,XMMWORD[16+rcx] ; remaining half of *userKey + mov edx,13 ; 14 rounds for 256 lea rax,[16+rax] movdqa xmm5,XMMWORD[$L$key_rotate] movdqa xmm4,XMMWORD[$L$key_rcon1] @@ -2451,7 +2451,7 @@ jmp NEAR $L$oop_key256 $L$done_key256: - mov DWORD[16+rax],edx + mov DWORD[16+rax],edx ; 240(%r8) xor eax,eax jmp NEAR $L$enc_key_ret_alt @@ -2516,29 +2516,29 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail - lea rsi,[rax] - lea rdi,[512+r8] - mov ecx,8 - DD 0xa548f3fc - lea rax,[88+rax] + lea rsi,[rax] ; %xmm save area + lea rdi,[512+r8] ; &context.Xmm6 + mov ecx,8 ; 4*sizeof(%xmm0)/sizeof(%rax) + DD 0xa548f3fc ; cld; rep movsq + lea rax,[88+rax] ; adjust stack pointer jmp NEAR $L$common_seh_tail @@ -2557,33 +2557,33 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail - mov rax,QWORD[208+r8] + mov rax,QWORD[208+r8] ; pull context->R11 - lea rsi,[((-168))+rax] - lea rdi,[512+r8] - mov ecx,20 - DD 0xa548f3fc + lea rsi,[((-168))+rax] ; %xmm save area + lea rdi,[512+r8] ; & context.Xmm6 + mov ecx,20 ; 10*sizeof(%xmm0)/sizeof(%rax) + DD 0xa548f3fc ; cld; rep movsq - mov rbp,QWORD[((-8))+rax] - mov QWORD[160+r8],rbp + mov rbp,QWORD[((-8))+rax] ; restore saved %rbp + mov QWORD[160+r8],rbp ; restore context->Rbp jmp NEAR $L$common_seh_tail @@ -2601,62 +2601,62 @@ pushfq sub rsp,64 - mov rax,QWORD[152+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp + mov rbx,QWORD[248+r8] ; pull context->Rip lea r10,[$L$cbc_decrypt_bulk] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<"prologue" label jb NEAR $L$common_seh_tail - mov rax,QWORD[120+r8] + mov rax,QWORD[120+r8] ; pull context->Rax lea r10,[$L$cbc_decrypt_body] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<cbc_decrypt_body jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp lea r10,[$L$cbc_ret] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip>="epilogue" label jae NEAR $L$common_seh_tail - lea rsi,[16+rax] - lea rdi,[512+r8] - mov ecx,20 - DD 0xa548f3fc + lea rsi,[16+rax] ; %xmm save area + lea rdi,[512+r8] ; &context.Xmm6 + mov ecx,20 ; 10*sizeof(%xmm0)/sizeof(%rax) + DD 0xa548f3fc ; cld; rep movsq - mov rax,QWORD[208+r8] + mov rax,QWORD[208+r8] ; pull context->R11 - mov rbp,QWORD[((-8))+rax] - mov QWORD[160+r8],rbp + mov rbp,QWORD[((-8))+rax] ; restore saved %rbp + mov QWORD[160+r8],rbp ; restore context->Rbp $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -2687,11 +2687,11 @@ $L$SEH_info_ecb: DB 9,0,0,0 DD ecb_ccm64_se_handler wrt ..imagebase - DD $L$ecb_enc_body wrt ..imagebase,$L$ecb_enc_ret wrt ..imagebase + DD $L$ecb_enc_body wrt ..imagebase,$L$ecb_enc_ret wrt ..imagebase ; HandlerData[] $L$SEH_info_ctr32: DB 9,0,0,0 DD ctr_xts_se_handler wrt ..imagebase - DD $L$ctr32_body wrt ..imagebase,$L$ctr32_epilogue wrt ..imagebase + DD $L$ctr32_body wrt ..imagebase,$L$ctr32_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_cbc: DB 9,0,0,0 DD cbc_se_handler wrt ..imagebase @@ -2709,7 +2709,7 @@ section .xdata ALIGN 4 $L$SEH_info_aes_hw_set_encrypt_key_base_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aes_hw_set_encrypt_key_base_3-$L$SEH_begin_aes_hw_set_encrypt_key_base_1 DB 1 DB 0 @@ -2718,7 +2718,7 @@ DW 0 $L$SEH_info_aes_hw_set_encrypt_key_alt_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_aes_hw_set_encrypt_key_alt_3-$L$SEH_begin_aes_hw_set_encrypt_key_alt_1 DB 1 DB 0
diff --git a/gen/bcm/ghash-ssse3-x86_64-apple.S b/gen/bcm/ghash-ssse3-x86_64-apple.S index f66e1fa..558fb6e 100644 --- a/gen/bcm/ghash-ssse3-x86_64-apple.S +++ b/gen/bcm/ghash-ssse3-x86_64-apple.S
@@ -6,10 +6,10 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) .text - - - - +// gcm_gmult_ssse3 multiplies |Xi| by |Htable| and writes the result to |Xi|. +// |Xi| is represented in GHASH's serialized byte representation. |Htable| is +// formatted as described above. +// void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]); .globl _gcm_gmult_ssse3 .private_extern _gcm_gmult_ssse3 @@ -22,18 +22,18 @@ movdqa L$reverse_bytes(%rip),%xmm10 movdqa L$low4_mask(%rip),%xmm2 - +// Reverse input bytes to deserialize. pshufb %xmm10,%xmm0 - +// Split each byte into low (%xmm0) and high (%xmm1) halves. movdqa %xmm2,%xmm1 pandn %xmm0,%xmm1 psrld $4,%xmm1 pand %xmm2,%xmm0 - - - +// Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note +// that, due to bit reversal, %xmm3 contains bits that fall off when +// right-shifting, not left-shifting. pxor %xmm2,%xmm2 pxor %xmm3,%xmm3 movq $5,%rax @@ -41,31 +41,31 @@ movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -74,46 +74,46 @@ subq $1,%rax jnz L$oop_row_1 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $5,%rax L$oop_row_2: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -122,46 +122,46 @@ subq $1,%rax jnz L$oop_row_2 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $6,%rax L$oop_row_3: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -170,21 +170,21 @@ subq $1,%rax jnz L$oop_row_3 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 - +// Store the result. Reverse bytes to serialize. pshufb %xmm10,%xmm2 movdqu %xmm2,(%rdi) - +// Zero any registers which contain secrets. pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -196,11 +196,11 @@ - - - - - +// gcm_ghash_ssse3 incorporates |len| bytes from |in| to |Xi|, using |Htable| as +// the key. It writes the result back to |Xi|. |Xi| is represented in GHASH's +// serialized byte representation. |Htable| is formatted as described above. +// void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, +// size_t len); .globl _gcm_ghash_ssse3 .private_extern _gcm_ghash_ssse3 @@ -214,62 +214,62 @@ movdqa L$reverse_bytes(%rip),%xmm10 movdqa L$low4_mask(%rip),%xmm11 - +// This function only processes whole blocks. andq $-16,%rcx - - +// Reverse input bytes to deserialize. We maintain the running +// total in %xmm0. pshufb %xmm10,%xmm0 - +// Iterate over each block. On entry to each iteration, %xmm3 is zero. pxor %xmm3,%xmm3 L$oop_ghash: - +// Incorporate the next block of input. movdqu (%rdx),%xmm1 - pshufb %xmm10,%xmm1 + pshufb %xmm10,%xmm1 // Reverse bytes. pxor %xmm1,%xmm0 - +// Split each byte into low (%xmm0) and high (%xmm1) halves. movdqa %xmm11,%xmm1 pandn %xmm0,%xmm1 psrld $4,%xmm1 pand %xmm11,%xmm0 - - - +// Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note +// that, due to bit reversal, %xmm3 contains bits that fall off when +// right-shifting, not left-shifting. pxor %xmm2,%xmm2 - +// %xmm3 is already zero at this point. movq $5,%rax L$oop_row_4: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -278,46 +278,46 @@ subq $1,%rax jnz L$oop_row_4 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $5,%rax L$oop_row_5: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -326,46 +326,46 @@ subq $1,%rax jnz L$oop_row_5 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $6,%rax L$oop_row_6: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -374,31 +374,31 @@ subq $1,%rax jnz L$oop_row_6 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movdqa %xmm2,%xmm0 - +// Rewind %rsi for the next iteration. leaq -256(%rsi),%rsi - +// Advance input and continue. leaq 16(%rdx),%rdx subq $16,%rcx jnz L$oop_ghash - +// Reverse bytes and store the result. pshufb %xmm10,%xmm0 movdqu %xmm0,(%rdi) - +// Zero any registers which contain secrets. pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -415,11 +415,11 @@ .p2align 4 ghash_ssse3_constants: - - +// .Lreverse_bytes is a permutation which, if applied with pshufb, reverses the +// bytes in an XMM register. L$reverse_bytes: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 - +// .Llow4_mask is an XMM mask which selects the low four bits of each byte. L$low4_mask: .quad 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f .text
diff --git a/gen/bcm/ghash-ssse3-x86_64-linux.S b/gen/bcm/ghash-ssse3-x86_64-linux.S index eea627a..52bae0d 100644 --- a/gen/bcm/ghash-ssse3-x86_64-linux.S +++ b/gen/bcm/ghash-ssse3-x86_64-linux.S
@@ -6,10 +6,10 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) .text - - - - +// gcm_gmult_ssse3 multiplies |Xi| by |Htable| and writes the result to |Xi|. +// |Xi| is represented in GHASH's serialized byte representation. |Htable| is +// formatted as described above. +// void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]); .type gcm_gmult_ssse3, @function .globl gcm_gmult_ssse3 .hidden gcm_gmult_ssse3 @@ -22,18 +22,18 @@ movdqa .Lreverse_bytes(%rip),%xmm10 movdqa .Llow4_mask(%rip),%xmm2 - +// Reverse input bytes to deserialize. pshufb %xmm10,%xmm0 - +// Split each byte into low (%xmm0) and high (%xmm1) halves. movdqa %xmm2,%xmm1 pandn %xmm0,%xmm1 psrld $4,%xmm1 pand %xmm2,%xmm0 - - - +// Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note +// that, due to bit reversal, %xmm3 contains bits that fall off when +// right-shifting, not left-shifting. pxor %xmm2,%xmm2 pxor %xmm3,%xmm3 movq $5,%rax @@ -41,31 +41,31 @@ movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -74,46 +74,46 @@ subq $1,%rax jnz .Loop_row_1 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $5,%rax .Loop_row_2: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -122,46 +122,46 @@ subq $1,%rax jnz .Loop_row_2 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $6,%rax .Loop_row_3: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -170,21 +170,21 @@ subq $1,%rax jnz .Loop_row_3 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 - +// Store the result. Reverse bytes to serialize. pshufb %xmm10,%xmm2 movdqu %xmm2,(%rdi) - +// Zero any registers which contain secrets. pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -196,11 +196,11 @@ .cfi_endproc .size gcm_gmult_ssse3,.-gcm_gmult_ssse3 - - - - - +// gcm_ghash_ssse3 incorporates |len| bytes from |in| to |Xi|, using |Htable| as +// the key. It writes the result back to |Xi|. |Xi| is represented in GHASH's +// serialized byte representation. |Htable| is formatted as described above. +// void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, +// size_t len); .type gcm_ghash_ssse3, @function .globl gcm_ghash_ssse3 .hidden gcm_ghash_ssse3 @@ -213,62 +213,62 @@ movdqa .Lreverse_bytes(%rip),%xmm10 movdqa .Llow4_mask(%rip),%xmm11 - +// This function only processes whole blocks. andq $-16,%rcx - - +// Reverse input bytes to deserialize. We maintain the running +// total in %xmm0. pshufb %xmm10,%xmm0 - +// Iterate over each block. On entry to each iteration, %xmm3 is zero. pxor %xmm3,%xmm3 .Loop_ghash: - +// Incorporate the next block of input. movdqu (%rdx),%xmm1 - pshufb %xmm10,%xmm1 + pshufb %xmm10,%xmm1 // Reverse bytes. pxor %xmm1,%xmm0 - +// Split each byte into low (%xmm0) and high (%xmm1) halves. movdqa %xmm11,%xmm1 pandn %xmm0,%xmm1 psrld $4,%xmm1 pand %xmm11,%xmm0 - - - +// Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note +// that, due to bit reversal, %xmm3 contains bits that fall off when +// right-shifting, not left-shifting. pxor %xmm2,%xmm2 - +// %xmm3 is already zero at this point. movq $5,%rax .Loop_row_4: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -277,46 +277,46 @@ subq $1,%rax jnz .Loop_row_4 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $5,%rax .Loop_row_5: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -325,46 +325,46 @@ subq $1,%rax jnz .Loop_row_5 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movq $6,%rax .Loop_row_6: movdqu (%rsi),%xmm4 leaq 16(%rsi),%rsi - +// Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa %xmm2,%xmm6 palignr $1,%xmm3,%xmm6 movdqa %xmm6,%xmm3 psrldq $1,%xmm2 - - - +// Load the next table row and index the low and high bits of the input. +// Note the low (respectively, high) half corresponds to more +// (respectively, less) significant coefficients. movdqa %xmm4,%xmm5 pshufb %xmm0,%xmm4 pshufb %xmm1,%xmm5 - +// Add the high half (%xmm5) without shifting. pxor %xmm5,%xmm2 - - +// Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +// add into the carry register (%xmm3). movdqa %xmm4,%xmm5 psllq $60,%xmm5 movdqa %xmm5,%xmm6 pslldq $8,%xmm6 pxor %xmm6,%xmm3 - +// Next, add into %xmm2. psrldq $8,%xmm5 pxor %xmm5,%xmm2 psrlq $4,%xmm4 @@ -373,31 +373,31 @@ subq $1,%rax jnz .Loop_row_6 - - - pxor %xmm3,%xmm2 +// Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +// x^7, so we shift and XOR four times. + pxor %xmm3,%xmm2 // x^0 = 0 psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^1 = x psrlq $1,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1) = x^2 psrlq $5,%xmm3 - pxor %xmm3,%xmm2 + pxor %xmm3,%xmm2 // x^(1+1+5) = x^7 pxor %xmm3,%xmm3 movdqa %xmm2,%xmm0 - +// Rewind %rsi for the next iteration. leaq -256(%rsi),%rsi - +// Advance input and continue. leaq 16(%rdx),%rdx subq $16,%rcx jnz .Loop_ghash - +// Reverse bytes and store the result. pshufb %xmm10,%xmm0 movdqu %xmm0,(%rdi) - +// Zero any registers which contain secrets. pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -414,11 +414,11 @@ .align 16 ghash_ssse3_constants: - - +// .Lreverse_bytes is a permutation which, if applied with pshufb, reverses the +// bytes in an XMM register. .Lreverse_bytes: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 - +// .Llow4_mask is an XMM mask which selects the low four bits of each byte. .Llow4_mask: .quad 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f .text
diff --git a/gen/bcm/ghash-ssse3-x86_64-win.asm b/gen/bcm/ghash-ssse3-x86_64-win.asm index 94af7d5..4794beb 100644 --- a/gen/bcm/ghash-ssse3-x86_64-win.asm +++ b/gen/bcm/ghash-ssse3-x86_64-win.asm
@@ -14,10 +14,10 @@ section .text code align=64 - - - - +; gcm_gmult_ssse3 multiplies |Xi| by |Htable| and writes the result to |Xi|. +; |Xi| is represented in GHASH's serialized byte representation. |Htable| is +; formatted as described above. +; void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]); global gcm_gmult_ssse3 ALIGN 16 @@ -36,18 +36,18 @@ movdqa xmm10,XMMWORD[$L$reverse_bytes] movdqa xmm2,XMMWORD[$L$low4_mask] - +; Reverse input bytes to deserialize. pshufb xmm0,xmm10 - +; Split each byte into low (%xmm0) and high (%xmm1) halves. movdqa xmm1,xmm2 pandn xmm1,xmm0 psrld xmm1,4 pand xmm0,xmm2 - - - +; Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note +; that, due to bit reversal, %xmm3 contains bits that fall off when +; right-shifting, not left-shifting. pxor xmm2,xmm2 pxor xmm3,xmm3 mov rax,5 @@ -55,31 +55,31 @@ movdqu xmm4,XMMWORD[rdx] lea rdx,[16+rdx] - +; Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa xmm6,xmm2 palignr xmm6,xmm3,1 movdqa xmm3,xmm6 psrldq xmm2,1 - - - +; Load the next table row and index the low and high bits of the input. +; Note the low (respectively, high) half corresponds to more +; (respectively, less) significant coefficients. movdqa xmm5,xmm4 pshufb xmm4,xmm0 pshufb xmm5,xmm1 - +; Add the high half (%xmm5) without shifting. pxor xmm2,xmm5 - - +; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +; add into the carry register (%xmm3). movdqa xmm5,xmm4 psllq xmm5,60 movdqa xmm6,xmm5 pslldq xmm6,8 pxor xmm3,xmm6 - +; Next, add into %xmm2. psrldq xmm5,8 pxor xmm2,xmm5 psrlq xmm4,4 @@ -88,46 +88,46 @@ sub rax,1 jnz NEAR $L$oop_row_1 - - - pxor xmm2,xmm3 +; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +; x^7, so we shift and XOR four times. + pxor xmm2,xmm3 ; x^0 = 0 psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^1 = x psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1) = x^2 psrlq xmm3,5 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1+5) = x^7 pxor xmm3,xmm3 mov rax,5 $L$oop_row_2: movdqu xmm4,XMMWORD[rdx] lea rdx,[16+rdx] - +; Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa xmm6,xmm2 palignr xmm6,xmm3,1 movdqa xmm3,xmm6 psrldq xmm2,1 - - - +; Load the next table row and index the low and high bits of the input. +; Note the low (respectively, high) half corresponds to more +; (respectively, less) significant coefficients. movdqa xmm5,xmm4 pshufb xmm4,xmm0 pshufb xmm5,xmm1 - +; Add the high half (%xmm5) without shifting. pxor xmm2,xmm5 - - +; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +; add into the carry register (%xmm3). movdqa xmm5,xmm4 psllq xmm5,60 movdqa xmm6,xmm5 pslldq xmm6,8 pxor xmm3,xmm6 - +; Next, add into %xmm2. psrldq xmm5,8 pxor xmm2,xmm5 psrlq xmm4,4 @@ -136,46 +136,46 @@ sub rax,1 jnz NEAR $L$oop_row_2 - - - pxor xmm2,xmm3 +; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +; x^7, so we shift and XOR four times. + pxor xmm2,xmm3 ; x^0 = 0 psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^1 = x psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1) = x^2 psrlq xmm3,5 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1+5) = x^7 pxor xmm3,xmm3 mov rax,6 $L$oop_row_3: movdqu xmm4,XMMWORD[rdx] lea rdx,[16+rdx] - +; Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa xmm6,xmm2 palignr xmm6,xmm3,1 movdqa xmm3,xmm6 psrldq xmm2,1 - - - +; Load the next table row and index the low and high bits of the input. +; Note the low (respectively, high) half corresponds to more +; (respectively, less) significant coefficients. movdqa xmm5,xmm4 pshufb xmm4,xmm0 pshufb xmm5,xmm1 - +; Add the high half (%xmm5) without shifting. pxor xmm2,xmm5 - - +; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +; add into the carry register (%xmm3). movdqa xmm5,xmm4 psllq xmm5,60 movdqa xmm6,xmm5 pslldq xmm6,8 pxor xmm3,xmm6 - +; Next, add into %xmm2. psrldq xmm5,8 pxor xmm2,xmm5 psrlq xmm4,4 @@ -184,21 +184,21 @@ sub rax,1 jnz NEAR $L$oop_row_3 - - - pxor xmm2,xmm3 +; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +; x^7, so we shift and XOR four times. + pxor xmm2,xmm3 ; x^0 = 0 psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^1 = x psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1) = x^2 psrlq xmm3,5 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1+5) = x^7 pxor xmm3,xmm3 - +; Store the result. Reverse bytes to serialize. pshufb xmm2,xmm10 movdqu XMMWORD[rcx],xmm2 - +; Zero any registers which contain secrets. pxor xmm0,xmm0 pxor xmm1,xmm1 pxor xmm2,xmm2 @@ -213,11 +213,11 @@ $L$SEH_end_gcm_gmult_ssse3_6: - - - - - +; gcm_ghash_ssse3 incorporates |len| bytes from |in| to |Xi|, using |Htable| as +; the key. It writes the result back to |Xi|. |Xi| is represented in GHASH's +; serialized byte representation. |Htable| is formatted as described above. +; void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, +; size_t len); global gcm_ghash_ssse3 ALIGN 16 @@ -238,62 +238,62 @@ movdqa xmm10,XMMWORD[$L$reverse_bytes] movdqa xmm11,XMMWORD[$L$low4_mask] - +; This function only processes whole blocks. and r9,-16 - - +; Reverse input bytes to deserialize. We maintain the running +; total in %xmm0. pshufb xmm0,xmm10 - +; Iterate over each block. On entry to each iteration, %xmm3 is zero. pxor xmm3,xmm3 $L$oop_ghash: - +; Incorporate the next block of input. movdqu xmm1,XMMWORD[r8] - pshufb xmm1,xmm10 + pshufb xmm1,xmm10 ; Reverse bytes. pxor xmm0,xmm1 - +; Split each byte into low (%xmm0) and high (%xmm1) halves. movdqa xmm1,xmm11 pandn xmm1,xmm0 psrld xmm1,4 pand xmm0,xmm11 - - - +; Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note +; that, due to bit reversal, %xmm3 contains bits that fall off when +; right-shifting, not left-shifting. pxor xmm2,xmm2 - +; %xmm3 is already zero at this point. mov rax,5 $L$oop_row_4: movdqu xmm4,XMMWORD[rdx] lea rdx,[16+rdx] - +; Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa xmm6,xmm2 palignr xmm6,xmm3,1 movdqa xmm3,xmm6 psrldq xmm2,1 - - - +; Load the next table row and index the low and high bits of the input. +; Note the low (respectively, high) half corresponds to more +; (respectively, less) significant coefficients. movdqa xmm5,xmm4 pshufb xmm4,xmm0 pshufb xmm5,xmm1 - +; Add the high half (%xmm5) without shifting. pxor xmm2,xmm5 - - +; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +; add into the carry register (%xmm3). movdqa xmm5,xmm4 psllq xmm5,60 movdqa xmm6,xmm5 pslldq xmm6,8 pxor xmm3,xmm6 - +; Next, add into %xmm2. psrldq xmm5,8 pxor xmm2,xmm5 psrlq xmm4,4 @@ -302,46 +302,46 @@ sub rax,1 jnz NEAR $L$oop_row_4 - - - pxor xmm2,xmm3 +; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +; x^7, so we shift and XOR four times. + pxor xmm2,xmm3 ; x^0 = 0 psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^1 = x psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1) = x^2 psrlq xmm3,5 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1+5) = x^7 pxor xmm3,xmm3 mov rax,5 $L$oop_row_5: movdqu xmm4,XMMWORD[rdx] lea rdx,[16+rdx] - +; Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa xmm6,xmm2 palignr xmm6,xmm3,1 movdqa xmm3,xmm6 psrldq xmm2,1 - - - +; Load the next table row and index the low and high bits of the input. +; Note the low (respectively, high) half corresponds to more +; (respectively, less) significant coefficients. movdqa xmm5,xmm4 pshufb xmm4,xmm0 pshufb xmm5,xmm1 - +; Add the high half (%xmm5) without shifting. pxor xmm2,xmm5 - - +; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +; add into the carry register (%xmm3). movdqa xmm5,xmm4 psllq xmm5,60 movdqa xmm6,xmm5 pslldq xmm6,8 pxor xmm3,xmm6 - +; Next, add into %xmm2. psrldq xmm5,8 pxor xmm2,xmm5 psrlq xmm4,4 @@ -350,46 +350,46 @@ sub rax,1 jnz NEAR $L$oop_row_5 - - - pxor xmm2,xmm3 +; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +; x^7, so we shift and XOR four times. + pxor xmm2,xmm3 ; x^0 = 0 psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^1 = x psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1) = x^2 psrlq xmm3,5 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1+5) = x^7 pxor xmm3,xmm3 mov rax,6 $L$oop_row_6: movdqu xmm4,XMMWORD[rdx] lea rdx,[16+rdx] - +; Right-shift %xmm2 and %xmm3 by 8 bytes. movdqa xmm6,xmm2 palignr xmm6,xmm3,1 movdqa xmm3,xmm6 psrldq xmm2,1 - - - +; Load the next table row and index the low and high bits of the input. +; Note the low (respectively, high) half corresponds to more +; (respectively, less) significant coefficients. movdqa xmm5,xmm4 pshufb xmm4,xmm0 pshufb xmm5,xmm1 - +; Add the high half (%xmm5) without shifting. pxor xmm2,xmm5 - - +; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, +; add into the carry register (%xmm3). movdqa xmm5,xmm4 psllq xmm5,60 movdqa xmm6,xmm5 pslldq xmm6,8 pxor xmm3,xmm6 - +; Next, add into %xmm2. psrldq xmm5,8 pxor xmm2,xmm5 psrlq xmm4,4 @@ -398,31 +398,31 @@ sub rax,1 jnz NEAR $L$oop_row_6 - - - pxor xmm2,xmm3 +; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + +; x^7, so we shift and XOR four times. + pxor xmm2,xmm3 ; x^0 = 0 psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^1 = x psrlq xmm3,1 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1) = x^2 psrlq xmm3,5 - pxor xmm2,xmm3 + pxor xmm2,xmm3 ; x^(1+1+5) = x^7 pxor xmm3,xmm3 movdqa xmm0,xmm2 - +; Rewind %rdx for the next iteration. lea rdx,[((-256))+rdx] - +; Advance input and continue. lea r8,[16+r8] sub r9,16 jnz NEAR $L$oop_ghash - +; Reverse bytes and store the result. pshufb xmm0,xmm10 movdqu XMMWORD[rcx],xmm0 - +; Zero any registers which contain secrets. pxor xmm0,xmm0 pxor xmm1,xmm1 pxor xmm2,xmm2 @@ -443,11 +443,11 @@ ALIGN 16 ghash_ssse3_constants: - - +; .Lreverse_bytes is a permutation which, if applied with pshufb, reverses the +; bytes in an XMM register. $L$reverse_bytes: DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 - +; .Llow4_mask is an XMM mask which selects the low four bits of each byte. $L$low4_mask: DQ 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f section .text @@ -466,7 +466,7 @@ section .xdata rdata align=8 ALIGN 4 $L$SEH_info_gcm_gmult_ssse3_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_gmult_ssse3_5-$L$SEH_begin_gcm_gmult_ssse3_1 DB 5 DB 0 @@ -481,7 +481,7 @@ DW 0 $L$SEH_info_gcm_ghash_ssse3_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_ghash_ssse3_6-$L$SEH_begin_gcm_ghash_ssse3_1 DB 7 DB 0
diff --git a/gen/bcm/ghash-x86_64-apple.S b/gen/bcm/ghash-x86_64-apple.S index e8d4745..0cfc20d 100644 --- a/gen/bcm/ghash-x86_64-apple.S +++ b/gen/bcm/ghash-x86_64-apple.S
@@ -15,154 +15,154 @@ _CET_ENDBR L$_init_clmul: movdqu (%rsi),%xmm2 - pshufd $78,%xmm2,%xmm2 + pshufd $78,%xmm2,%xmm2 // dword swap - - pshufd $255,%xmm2,%xmm4 +// <<1 twist + pshufd $255,%xmm2,%xmm4 // broadcast uppermost dword movdqa %xmm2,%xmm3 psllq $1,%xmm2 - pxor %xmm5,%xmm5 + pxor %xmm5,%xmm5 // psrlq $63,%xmm3 - pcmpgtd %xmm4,%xmm5 + pcmpgtd %xmm4,%xmm5 // broadcast carry bit pslldq $8,%xmm3 - por %xmm3,%xmm2 + por %xmm3,%xmm2 // H<<=1 - +// magic reduction pand L$0x1c2_polynomial(%rip),%xmm5 - pxor %xmm5,%xmm2 + pxor %xmm5,%xmm2 // if(carry) H^=0x1c2_polynomial - +// calculate H^2 pshufd $78,%xmm2,%xmm6 movdqa %xmm2,%xmm0 pxor %xmm2,%xmm6 - movdqa %xmm0,%xmm1 + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm6,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm6,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // pshufd $78,%xmm2,%xmm3 pshufd $78,%xmm0,%xmm4 - pxor %xmm2,%xmm3 - movdqu %xmm2,0(%rdi) - pxor %xmm0,%xmm4 - movdqu %xmm0,16(%rdi) - palignr $8,%xmm3,%xmm4 - movdqu %xmm4,32(%rdi) - movdqa %xmm0,%xmm1 + pxor %xmm2,%xmm3 // Karatsuba pre-processing + movdqu %xmm2,0(%rdi) // save H + pxor %xmm0,%xmm4 // Karatsuba pre-processing + movdqu %xmm0,16(%rdi) // save H^2 + palignr $8,%xmm3,%xmm4 // low part is H.lo^H.hi... + movdqu %xmm4,32(%rdi) // save Karatsuba "salt" + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm6,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm6,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // movdqa %xmm0,%xmm5 - movdqa %xmm0,%xmm1 + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm6,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm6,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // pshufd $78,%xmm5,%xmm3 pshufd $78,%xmm0,%xmm4 - pxor %xmm5,%xmm3 - movdqu %xmm5,48(%rdi) - pxor %xmm0,%xmm4 - movdqu %xmm0,64(%rdi) - palignr $8,%xmm3,%xmm4 - movdqu %xmm4,80(%rdi) + pxor %xmm5,%xmm3 // Karatsuba pre-processing + movdqu %xmm5,48(%rdi) // save H^3 + pxor %xmm0,%xmm4 // Karatsuba pre-processing + movdqu %xmm0,64(%rdi) // save H^4 + palignr $8,%xmm3,%xmm4 // low part is H^3.lo^H^3.hi... + movdqu %xmm4,80(%rdi) // save Karatsuba "salt" ret @@ -181,43 +181,43 @@ movdqu (%rsi),%xmm2 movdqu 32(%rsi),%xmm4 pshufb %xmm5,%xmm0 - movdqa %xmm0,%xmm1 + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm4,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm4,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // pshufb %xmm5,%xmm0 movdqu %xmm0,(%rdi) ret @@ -248,13 +248,13 @@ jb L$skip4x subq $0x30,%rcx - movq $0xA040608020C0E000,%rax + movq $0xA040608020C0E000,%rax // ((7..0)·0xE0)&0xff movdqu 48(%rsi),%xmm14 movdqu 64(%rsi),%xmm15 - - - +// ###### +// Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P +// movdqu 48(%rdx),%xmm3 movdqu 32(%rdx),%xmm11 pshufb %xmm10,%xmm3 @@ -319,33 +319,33 @@ pclmulqdq $0x00,%xmm2,%xmm11 pshufd $78,%xmm3,%xmm4 - pxor %xmm0,%xmm8 + pxor %xmm0,%xmm8 // aggregated Karatsuba post-processing movdqa %xmm3,%xmm5 - pxor %xmm1,%xmm8 + pxor %xmm1,%xmm8 // pxor %xmm3,%xmm4 - movdqa %xmm8,%xmm9 + movdqa %xmm8,%xmm9 // pclmulqdq $0x11,%xmm2,%xmm13 pslldq $8,%xmm8 - psrldq $8,%xmm9 + psrldq $8,%xmm9 // pxor %xmm8,%xmm0 movdqa L$7_mask(%rip),%xmm8 - pxor %xmm9,%xmm1 + pxor %xmm9,%xmm1 // movq %rax,%xmm9 - pand %xmm0,%xmm8 - pshufb %xmm8,%xmm9 - pxor %xmm0,%xmm9 + pand %xmm0,%xmm8 // 1st phase + pshufb %xmm8,%xmm9 // + pxor %xmm0,%xmm9 // pclmulqdq $0x00,%xmm7,%xmm12 - psllq $57,%xmm9 - movdqa %xmm9,%xmm8 + psllq $57,%xmm9 // + movdqa %xmm9,%xmm8 // pslldq $8,%xmm9 pclmulqdq $0x00,%xmm6,%xmm3 - psrldq $8,%xmm8 + psrldq $8,%xmm8 // pxor %xmm9,%xmm0 - pxor %xmm8,%xmm1 + pxor %xmm8,%xmm1 // movdqu 0(%rdx),%xmm8 - movdqa %xmm0,%xmm9 + movdqa %xmm0,%xmm9 // 2nd phase psrlq $1,%xmm0 pclmulqdq $0x11,%xmm6,%xmm5 xorps %xmm11,%xmm3 @@ -355,19 +355,19 @@ xorps %xmm13,%xmm5 movups 80(%rsi),%xmm7 pshufb %xmm10,%xmm8 - pxor %xmm9,%xmm1 + pxor %xmm9,%xmm1 // pxor %xmm0,%xmm9 psrlq $5,%xmm0 movdqa %xmm11,%xmm13 pxor %xmm12,%xmm4 pshufd $78,%xmm11,%xmm12 - pxor %xmm9,%xmm0 + pxor %xmm9,%xmm0 // pxor %xmm8,%xmm1 pxor %xmm11,%xmm12 pclmulqdq $0x00,%xmm14,%xmm11 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // movdqa %xmm0,%xmm1 pclmulqdq $0x11,%xmm14,%xmm13 xorps %xmm11,%xmm3 @@ -388,56 +388,56 @@ xorps %xmm12,%xmm4 xorps %xmm3,%xmm0 xorps %xmm5,%xmm1 - pxor %xmm0,%xmm1 + pxor %xmm0,%xmm1 // aggregated Karatsuba post-processing pxor %xmm4,%xmm8 - pxor %xmm1,%xmm8 + pxor %xmm1,%xmm8 // pxor %xmm0,%xmm1 - movdqa %xmm8,%xmm9 + movdqa %xmm8,%xmm9 // psrldq $8,%xmm8 - pslldq $8,%xmm9 + pslldq $8,%xmm9 // pxor %xmm8,%xmm1 - pxor %xmm9,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm9,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // addq $0x40,%rcx jz L$done movdqu 32(%rsi),%xmm7 subq $0x10,%rcx jz L$odd_tail L$skip4x: - - - - - - movdqu (%rdx),%xmm8 - movdqu 16(%rdx),%xmm3 +// ###### +// Xi+2 =[H*(Ii+1 + Xi+1)] mod P = +// [(H*Ii+1) + (H*Xi+1)] mod P = +// [(H*Ii+1) + H^2*(Ii+Xi)] mod P +// + movdqu (%rdx),%xmm8 // Ii + movdqu 16(%rdx),%xmm3 // Ii+1 pshufb %xmm10,%xmm8 pshufb %xmm10,%xmm3 - pxor %xmm8,%xmm0 + pxor %xmm8,%xmm0 // Ii+Xi movdqa %xmm3,%xmm5 pshufd $78,%xmm3,%xmm4 @@ -446,7 +446,7 @@ pclmulqdq $0x11,%xmm2,%xmm5 pclmulqdq $0x00,%xmm7,%xmm4 - leaq 32(%rdx),%rdx + leaq 32(%rdx),%rdx // i+=2 nop subq $0x20,%rcx jbe L$even_tail @@ -457,59 +457,59 @@ L$mod_loop: movdqa %xmm0,%xmm1 movdqa %xmm4,%xmm8 - pshufd $78,%xmm0,%xmm4 - pxor %xmm0,%xmm4 + pshufd $78,%xmm0,%xmm4 // + pxor %xmm0,%xmm4 // pclmulqdq $0x00,%xmm6,%xmm0 pclmulqdq $0x11,%xmm6,%xmm1 pclmulqdq $0x10,%xmm7,%xmm4 - pxor %xmm3,%xmm0 + pxor %xmm3,%xmm0 // (H*Ii+1) + H^2*(Ii+Xi) pxor %xmm5,%xmm1 - movdqu (%rdx),%xmm9 - pxor %xmm0,%xmm8 + movdqu (%rdx),%xmm9 // Ii + pxor %xmm0,%xmm8 // aggregated Karatsuba post-processing pshufb %xmm10,%xmm9 - movdqu 16(%rdx),%xmm3 + movdqu 16(%rdx),%xmm3 // Ii+1 pxor %xmm1,%xmm8 - pxor %xmm9,%xmm1 + pxor %xmm9,%xmm1 // "Ii+Xi", consume early pxor %xmm8,%xmm4 pshufb %xmm10,%xmm3 - movdqa %xmm4,%xmm8 + movdqa %xmm4,%xmm8 // psrldq $8,%xmm8 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm8,%xmm1 - pxor %xmm4,%xmm0 + pxor %xmm4,%xmm0 // - movdqa %xmm3,%xmm5 + movdqa %xmm3,%xmm5 // - movdqa %xmm0,%xmm9 + movdqa %xmm0,%xmm9 // 1st phase movdqa %xmm0,%xmm8 psllq $5,%xmm0 - pxor %xmm0,%xmm8 - pclmulqdq $0x00,%xmm2,%xmm3 + pxor %xmm0,%xmm8 // + pclmulqdq $0x00,%xmm2,%xmm3 // ###### psllq $1,%xmm0 - pxor %xmm8,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm8 + pxor %xmm8,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm8 // pslldq $8,%xmm0 - psrldq $8,%xmm8 + psrldq $8,%xmm8 // pxor %xmm9,%xmm0 pshufd $78,%xmm5,%xmm4 - pxor %xmm8,%xmm1 - pxor %xmm5,%xmm4 + pxor %xmm8,%xmm1 // + pxor %xmm5,%xmm4 // - movdqa %xmm0,%xmm9 + movdqa %xmm0,%xmm9 // 2nd phase psrlq $1,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm5 - pxor %xmm9,%xmm1 + pclmulqdq $0x11,%xmm2,%xmm5 // ###### + pxor %xmm9,%xmm1 // pxor %xmm0,%xmm9 psrlq $5,%xmm0 - pxor %xmm9,%xmm0 + pxor %xmm9,%xmm0 // leaq 32(%rdx),%rdx - psrlq $1,%xmm0 - pclmulqdq $0x00,%xmm7,%xmm4 - pxor %xmm1,%xmm0 + psrlq $1,%xmm0 // + pclmulqdq $0x00,%xmm7,%xmm4 // ###### + pxor %xmm1,%xmm0 // subq $0x20,%rcx ja L$mod_loop @@ -517,90 +517,90 @@ L$even_tail: movdqa %xmm0,%xmm1 movdqa %xmm4,%xmm8 - pshufd $78,%xmm0,%xmm4 - pxor %xmm0,%xmm4 + pshufd $78,%xmm0,%xmm4 // + pxor %xmm0,%xmm4 // pclmulqdq $0x00,%xmm6,%xmm0 pclmulqdq $0x11,%xmm6,%xmm1 pclmulqdq $0x10,%xmm7,%xmm4 - pxor %xmm3,%xmm0 + pxor %xmm3,%xmm0 // (H*Ii+1) + H^2*(Ii+Xi) pxor %xmm5,%xmm1 pxor %xmm0,%xmm8 pxor %xmm1,%xmm8 pxor %xmm8,%xmm4 - movdqa %xmm4,%xmm8 + movdqa %xmm4,%xmm8 // psrldq $8,%xmm8 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm8,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // testq %rcx,%rcx jnz L$done L$odd_tail: - movdqu (%rdx),%xmm8 + movdqu (%rdx),%xmm8 // Ii pshufb %xmm10,%xmm8 - pxor %xmm8,%xmm0 - movdqa %xmm0,%xmm1 + pxor %xmm8,%xmm0 // Ii+Xi + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm7,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm7,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // L$done: pshufb %xmm10,%xmm0 movdqu %xmm0,(%rdi) @@ -620,101 +620,101 @@ vzeroupper vmovdqu (%rsi),%xmm2 - vpshufd $78,%xmm2,%xmm2 + vpshufd $78,%xmm2,%xmm2 // dword swap - - vpshufd $255,%xmm2,%xmm4 +// <<1 twist + vpshufd $255,%xmm2,%xmm4 // broadcast uppermost dword vpsrlq $63,%xmm2,%xmm3 vpsllq $1,%xmm2,%xmm2 - vpxor %xmm5,%xmm5,%xmm5 - vpcmpgtd %xmm4,%xmm5,%xmm5 + vpxor %xmm5,%xmm5,%xmm5 // + vpcmpgtd %xmm4,%xmm5,%xmm5 // broadcast carry bit vpslldq $8,%xmm3,%xmm3 - vpor %xmm3,%xmm2,%xmm2 + vpor %xmm3,%xmm2,%xmm2 // H<<=1 - +// magic reduction vpand L$0x1c2_polynomial(%rip),%xmm5,%xmm5 - vpxor %xmm5,%xmm2,%xmm2 + vpxor %xmm5,%xmm2,%xmm2 // if(carry) H^=0x1c2_polynomial vpunpckhqdq %xmm2,%xmm2,%xmm6 vmovdqa %xmm2,%xmm0 vpxor %xmm2,%xmm6,%xmm6 - movq $4,%r10 + movq $4,%r10 // up to H^8 jmp L$init_start_avx .p2align 5 L$init_loop_avx: - vpalignr $8,%xmm3,%xmm4,%xmm5 - vmovdqu %xmm5,-16(%rdi) + vpalignr $8,%xmm3,%xmm4,%xmm5 // low part is H.lo^H.hi... + vmovdqu %xmm5,-16(%rdi) // save Karatsuba "salt" vpunpckhqdq %xmm0,%xmm0,%xmm3 - vpxor %xmm0,%xmm3,%xmm3 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 - vpxor %xmm0,%xmm1,%xmm4 - vpxor %xmm4,%xmm3,%xmm3 + vpxor %xmm0,%xmm3,%xmm3 // + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 // ###### + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 // ###### + vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 // ###### + vpxor %xmm0,%xmm1,%xmm4 // + vpxor %xmm4,%xmm3,%xmm3 // - vpslldq $8,%xmm3,%xmm4 + vpslldq $8,%xmm3,%xmm4 // vpsrldq $8,%xmm3,%xmm3 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpxor %xmm3,%xmm1,%xmm1 - vpsllq $57,%xmm0,%xmm3 + vpsllq $57,%xmm0,%xmm3 // 1st phase vpsllq $62,%xmm0,%xmm4 - vpxor %xmm3,%xmm4,%xmm4 + vpxor %xmm3,%xmm4,%xmm4 // vpsllq $63,%xmm0,%xmm3 - vpxor %xmm3,%xmm4,%xmm4 - vpslldq $8,%xmm4,%xmm3 + vpxor %xmm3,%xmm4,%xmm4 // + vpslldq $8,%xmm4,%xmm3 // vpsrldq $8,%xmm4,%xmm4 - vpxor %xmm3,%xmm0,%xmm0 + vpxor %xmm3,%xmm0,%xmm0 // vpxor %xmm4,%xmm1,%xmm1 - vpsrlq $1,%xmm0,%xmm4 + vpsrlq $1,%xmm0,%xmm4 // 2nd phase vpxor %xmm0,%xmm1,%xmm1 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpsrlq $5,%xmm4,%xmm4 - vpxor %xmm4,%xmm0,%xmm0 - vpsrlq $1,%xmm0,%xmm0 - vpxor %xmm1,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // + vpsrlq $1,%xmm0,%xmm0 // + vpxor %xmm1,%xmm0,%xmm0 // L$init_start_avx: vmovdqa %xmm0,%xmm5 vpunpckhqdq %xmm0,%xmm0,%xmm3 - vpxor %xmm0,%xmm3,%xmm3 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 - vpxor %xmm0,%xmm1,%xmm4 - vpxor %xmm4,%xmm3,%xmm3 + vpxor %xmm0,%xmm3,%xmm3 // + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 // ###### + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 // ###### + vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 // ###### + vpxor %xmm0,%xmm1,%xmm4 // + vpxor %xmm4,%xmm3,%xmm3 // - vpslldq $8,%xmm3,%xmm4 + vpslldq $8,%xmm3,%xmm4 // vpsrldq $8,%xmm3,%xmm3 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpxor %xmm3,%xmm1,%xmm1 - vpsllq $57,%xmm0,%xmm3 + vpsllq $57,%xmm0,%xmm3 // 1st phase vpsllq $62,%xmm0,%xmm4 - vpxor %xmm3,%xmm4,%xmm4 + vpxor %xmm3,%xmm4,%xmm4 // vpsllq $63,%xmm0,%xmm3 - vpxor %xmm3,%xmm4,%xmm4 - vpslldq $8,%xmm4,%xmm3 + vpxor %xmm3,%xmm4,%xmm4 // + vpslldq $8,%xmm4,%xmm3 // vpsrldq $8,%xmm4,%xmm4 - vpxor %xmm3,%xmm0,%xmm0 + vpxor %xmm3,%xmm0,%xmm0 // vpxor %xmm4,%xmm1,%xmm1 - vpsrlq $1,%xmm0,%xmm4 + vpsrlq $1,%xmm0,%xmm4 // 2nd phase vpxor %xmm0,%xmm1,%xmm1 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpsrlq $5,%xmm4,%xmm4 - vpxor %xmm4,%xmm0,%xmm0 - vpsrlq $1,%xmm0,%xmm0 - vpxor %xmm1,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // + vpsrlq $1,%xmm0,%xmm0 // + vpxor %xmm1,%xmm0,%xmm0 // vpshufd $78,%xmm5,%xmm3 vpshufd $78,%xmm0,%xmm4 - vpxor %xmm5,%xmm3,%xmm3 - vmovdqu %xmm5,0(%rdi) - vpxor %xmm0,%xmm4,%xmm4 - vmovdqu %xmm0,16(%rdi) + vpxor %xmm5,%xmm3,%xmm3 // Karatsuba pre-processing + vmovdqu %xmm5,0(%rdi) // save H^1,3,5,7 + vpxor %xmm0,%xmm4,%xmm4 // Karatsuba pre-processing + vmovdqu %xmm0,16(%rdi) // save H^2,4,6,8 leaq 48(%rdi),%rdi subq $1,%r10 jnz L$init_loop_avx - vpalignr $8,%xmm4,%xmm3,%xmm5 + vpalignr $8,%xmm4,%xmm3,%xmm5 // last "salt" is flipped vmovdqu %xmm5,-16(%rdi) vzeroupper @@ -744,29 +744,29 @@ _CET_ENDBR vzeroupper - vmovdqu (%rdi),%xmm10 + vmovdqu (%rdi),%xmm10 // load %xmm10 leaq L$0x1c2_polynomial(%rip),%r10 - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // size optimization vmovdqu L$bswap_mask(%rip),%xmm13 vpshufb %xmm13,%xmm10,%xmm10 cmpq $0x80,%rcx jb L$short_avx subq $0x80,%rcx - vmovdqu 112(%rdx),%xmm14 - vmovdqu 0-64(%rsi),%xmm6 + vmovdqu 112(%rdx),%xmm14 // I[7] + vmovdqu 0-64(%rsi),%xmm6 // %xmm6^1 vpshufb %xmm13,%xmm14,%xmm14 vmovdqu 32-64(%rsi),%xmm7 vpunpckhqdq %xmm14,%xmm14,%xmm9 - vmovdqu 96(%rdx),%xmm15 + vmovdqu 96(%rdx),%xmm15 // I[6] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpxor %xmm14,%xmm9,%xmm9 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 16-64(%rsi),%xmm6 + vmovdqu 16-64(%rsi),%xmm6 // %xmm6^2 vpunpckhqdq %xmm15,%xmm15,%xmm8 - vmovdqu 80(%rdx),%xmm14 + vmovdqu 80(%rdx),%xmm14 // I[5] vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm15,%xmm8,%xmm8 @@ -774,9 +774,9 @@ vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 48-64(%rsi),%xmm6 + vmovdqu 48-64(%rsi),%xmm6 // %xmm6^3 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu 64(%rdx),%xmm15 + vmovdqu 64(%rdx),%xmm15 // I[4] vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 vmovdqu 80-64(%rsi),%xmm7 @@ -786,56 +786,56 @@ vpxor %xmm1,%xmm4,%xmm4 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 64-64(%rsi),%xmm6 + vmovdqu 64-64(%rsi),%xmm6 // %xmm6^4 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm15,%xmm8,%xmm8 - vmovdqu 48(%rdx),%xmm14 + vmovdqu 48(%rdx),%xmm14 // I[3] vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpxor %xmm4,%xmm1,%xmm1 vpshufb %xmm13,%xmm14,%xmm14 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 96-64(%rsi),%xmm6 + vmovdqu 96-64(%rsi),%xmm6 // %xmm6^5 vpxor %xmm5,%xmm2,%xmm2 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 vmovdqu 128-64(%rsi),%xmm7 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu 32(%rdx),%xmm15 + vmovdqu 32(%rdx),%xmm15 // I[2] vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpxor %xmm1,%xmm4,%xmm4 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 112-64(%rsi),%xmm6 + vmovdqu 112-64(%rsi),%xmm6 // %xmm6^6 vpxor %xmm2,%xmm5,%xmm5 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm15,%xmm8,%xmm8 - vmovdqu 16(%rdx),%xmm14 + vmovdqu 16(%rdx),%xmm14 // I[1] vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpxor %xmm4,%xmm1,%xmm1 vpshufb %xmm13,%xmm14,%xmm14 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 144-64(%rsi),%xmm6 + vmovdqu 144-64(%rsi),%xmm6 // %xmm6^7 vpxor %xmm5,%xmm2,%xmm2 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 vmovdqu 176-64(%rsi),%xmm7 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu (%rdx),%xmm15 + vmovdqu (%rdx),%xmm15 // I[0] vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpxor %xmm1,%xmm4,%xmm4 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 160-64(%rsi),%xmm6 + vmovdqu 160-64(%rsi),%xmm6 // %xmm6^8 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x10,%xmm7,%xmm9,%xmm2 @@ -843,41 +843,41 @@ cmpq $0x80,%rcx jb L$tail_avx - vpxor %xmm10,%xmm15,%xmm15 + vpxor %xmm10,%xmm15,%xmm15 // accumulate %xmm10 subq $0x80,%rcx jmp L$oop8x_avx .p2align 5 L$oop8x_avx: vpunpckhqdq %xmm15,%xmm15,%xmm8 - vmovdqu 112(%rdx),%xmm14 + vmovdqu 112(%rdx),%xmm14 // I[7] vpxor %xmm0,%xmm3,%xmm3 vpxor %xmm15,%xmm8,%xmm8 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm10 vpshufb %xmm13,%xmm14,%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm11 - vmovdqu 0-64(%rsi),%xmm6 + vmovdqu 0-64(%rsi),%xmm6 // %xmm6^1 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm12 vmovdqu 32-64(%rsi),%xmm7 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu 96(%rdx),%xmm15 + vmovdqu 96(%rdx),%xmm15 // I[6] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 - vpxor %xmm3,%xmm10,%xmm10 + vpxor %xmm3,%xmm10,%xmm10 // collect result vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 vxorps %xmm4,%xmm11,%xmm11 - vmovdqu 16-64(%rsi),%xmm6 + vmovdqu 16-64(%rsi),%xmm6 // %xmm6^2 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm5,%xmm12,%xmm12 vxorps %xmm15,%xmm8,%xmm8 - vmovdqu 80(%rdx),%xmm14 - vpxor %xmm10,%xmm12,%xmm12 + vmovdqu 80(%rdx),%xmm14 // I[5] + vpxor %xmm10,%xmm12,%xmm12 // aggregated Karatsuba post-processing vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpxor %xmm11,%xmm12,%xmm12 vpslldq $8,%xmm12,%xmm9 @@ -885,7 +885,7 @@ vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 vpsrldq $8,%xmm12,%xmm12 vpxor %xmm9,%xmm10,%xmm10 - vmovdqu 48-64(%rsi),%xmm6 + vmovdqu 48-64(%rsi),%xmm6 // %xmm6^3 vpshufb %xmm13,%xmm14,%xmm14 vxorps %xmm12,%xmm11,%xmm11 vpxor %xmm1,%xmm4,%xmm4 @@ -895,26 +895,26 @@ vpxor %xmm14,%xmm9,%xmm9 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu 64(%rdx),%xmm15 - vpalignr $8,%xmm10,%xmm10,%xmm12 + vmovdqu 64(%rdx),%xmm15 // I[4] + vpalignr $8,%xmm10,%xmm10,%xmm12 // 1st phase vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpshufb %xmm13,%xmm15,%xmm15 vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 64-64(%rsi),%xmm6 + vmovdqu 64-64(%rsi),%xmm6 // %xmm6^4 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpxor %xmm4,%xmm1,%xmm1 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vxorps %xmm15,%xmm8,%xmm8 vpxor %xmm5,%xmm2,%xmm2 - vmovdqu 48(%rdx),%xmm14 + vmovdqu 48(%rdx),%xmm14 // I[3] vpclmulqdq $0x10,(%r10),%xmm10,%xmm10 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpshufb %xmm13,%xmm14,%xmm14 vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 96-64(%rsi),%xmm6 + vmovdqu 96-64(%rsi),%xmm6 // %xmm6^5 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 @@ -922,12 +922,12 @@ vpxor %xmm14,%xmm9,%xmm9 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu 32(%rdx),%xmm15 + vmovdqu 32(%rdx),%xmm15 // I[2] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpshufb %xmm13,%xmm15,%xmm15 vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 112-64(%rsi),%xmm6 + vmovdqu 112-64(%rsi),%xmm6 // %xmm6^6 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpxor %xmm4,%xmm1,%xmm1 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 @@ -935,13 +935,13 @@ vpxor %xmm5,%xmm2,%xmm2 vxorps %xmm12,%xmm10,%xmm10 - vmovdqu 16(%rdx),%xmm14 - vpalignr $8,%xmm10,%xmm10,%xmm12 + vmovdqu 16(%rdx),%xmm14 // I[1] + vpalignr $8,%xmm10,%xmm10,%xmm12 // 2nd phase vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpshufb %xmm13,%xmm14,%xmm14 vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 144-64(%rsi),%xmm6 + vmovdqu 144-64(%rsi),%xmm6 // %xmm6^7 vpclmulqdq $0x10,(%r10),%xmm10,%xmm10 vxorps %xmm11,%xmm12,%xmm12 vpunpckhqdq %xmm14,%xmm14,%xmm9 @@ -951,14 +951,14 @@ vpxor %xmm14,%xmm9,%xmm9 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu (%rdx),%xmm15 + vmovdqu (%rdx),%xmm15 // I[0] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 160-64(%rsi),%xmm6 + vmovdqu 160-64(%rsi),%xmm6 // %xmm6^8 vpxor %xmm12,%xmm15,%xmm15 vpclmulqdq $0x10,%xmm7,%xmm9,%xmm2 - vpxor %xmm10,%xmm15,%xmm15 + vpxor %xmm10,%xmm15,%xmm15 // accumulate %xmm10 leaq 128(%rdx),%rdx subq $0x80,%rcx @@ -969,15 +969,15 @@ .p2align 5 L$short_avx: - vmovdqu -16(%rdx,%rcx,1),%xmm14 + vmovdqu -16(%rdx,%rcx,1),%xmm14 // very last word leaq (%rdx,%rcx,1),%rdx - vmovdqu 0-64(%rsi),%xmm6 + vmovdqu 0-64(%rsi),%xmm6 // %xmm6^1 vmovdqu 32-64(%rsi),%xmm7 vpshufb %xmm13,%xmm14,%xmm15 - vmovdqa %xmm0,%xmm3 - vmovdqa %xmm1,%xmm4 - vmovdqa %xmm2,%xmm5 + vmovdqa %xmm0,%xmm3 // subtle way to zero %xmm3, + vmovdqa %xmm1,%xmm4 // %xmm4 and + vmovdqa %xmm2,%xmm5 // %xmm5 subq $0x10,%rcx jz L$tail_avx @@ -988,7 +988,7 @@ vmovdqu -32(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 16-64(%rsi),%xmm6 + vmovdqu 16-64(%rsi),%xmm6 // %xmm6^2 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1003,7 +1003,7 @@ vmovdqu -48(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 48-64(%rsi),%xmm6 + vmovdqu 48-64(%rsi),%xmm6 // %xmm6^3 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1018,7 +1018,7 @@ vmovdqu -64(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 64-64(%rsi),%xmm6 + vmovdqu 64-64(%rsi),%xmm6 // %xmm6^4 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1033,7 +1033,7 @@ vmovdqu -80(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 96-64(%rsi),%xmm6 + vmovdqu 96-64(%rsi),%xmm6 // %xmm6^5 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1048,7 +1048,7 @@ vmovdqu -96(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 112-64(%rsi),%xmm6 + vmovdqu 112-64(%rsi),%xmm6 // %xmm6^6 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1063,7 +1063,7 @@ vmovdqu -112(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 144-64(%rsi),%xmm6 + vmovdqu 144-64(%rsi),%xmm6 // %xmm6^7 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1073,7 +1073,7 @@ .p2align 5 L$tail_avx: - vpxor %xmm10,%xmm15,%xmm15 + vpxor %xmm10,%xmm15,%xmm15 // accumulate %xmm10 L$tail_no_xor_avx: vpunpckhqdq %xmm15,%xmm15,%xmm8 vpxor %xmm0,%xmm3,%xmm3 @@ -1090,18 +1090,18 @@ vpxor %xmm1,%xmm4,%xmm11 vpxor %xmm2,%xmm5,%xmm5 - vpxor %xmm10,%xmm5,%xmm5 + vpxor %xmm10,%xmm5,%xmm5 // aggregated Karatsuba post-processing vpxor %xmm11,%xmm5,%xmm5 vpslldq $8,%xmm5,%xmm9 vpsrldq $8,%xmm5,%xmm5 vpxor %xmm9,%xmm10,%xmm10 vpxor %xmm5,%xmm11,%xmm11 - vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 + vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 // 1st phase vpalignr $8,%xmm10,%xmm10,%xmm10 vpxor %xmm9,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 + vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 // 2nd phase vpalignr $8,%xmm10,%xmm10,%xmm10 vpxor %xmm11,%xmm10,%xmm10 vpxor %xmm9,%xmm10,%xmm10
diff --git a/gen/bcm/ghash-x86_64-linux.S b/gen/bcm/ghash-x86_64-linux.S index 64d4cbe..6cfec7f 100644 --- a/gen/bcm/ghash-x86_64-linux.S +++ b/gen/bcm/ghash-x86_64-linux.S
@@ -15,154 +15,154 @@ _CET_ENDBR .L_init_clmul: movdqu (%rsi),%xmm2 - pshufd $78,%xmm2,%xmm2 + pshufd $78,%xmm2,%xmm2 // dword swap - - pshufd $255,%xmm2,%xmm4 +// <<1 twist + pshufd $255,%xmm2,%xmm4 // broadcast uppermost dword movdqa %xmm2,%xmm3 psllq $1,%xmm2 - pxor %xmm5,%xmm5 + pxor %xmm5,%xmm5 // psrlq $63,%xmm3 - pcmpgtd %xmm4,%xmm5 + pcmpgtd %xmm4,%xmm5 // broadcast carry bit pslldq $8,%xmm3 - por %xmm3,%xmm2 + por %xmm3,%xmm2 // H<<=1 - +// magic reduction pand .L0x1c2_polynomial(%rip),%xmm5 - pxor %xmm5,%xmm2 + pxor %xmm5,%xmm2 // if(carry) H^=0x1c2_polynomial - +// calculate H^2 pshufd $78,%xmm2,%xmm6 movdqa %xmm2,%xmm0 pxor %xmm2,%xmm6 - movdqa %xmm0,%xmm1 + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm6,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm6,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // pshufd $78,%xmm2,%xmm3 pshufd $78,%xmm0,%xmm4 - pxor %xmm2,%xmm3 - movdqu %xmm2,0(%rdi) - pxor %xmm0,%xmm4 - movdqu %xmm0,16(%rdi) - palignr $8,%xmm3,%xmm4 - movdqu %xmm4,32(%rdi) - movdqa %xmm0,%xmm1 + pxor %xmm2,%xmm3 // Karatsuba pre-processing + movdqu %xmm2,0(%rdi) // save H + pxor %xmm0,%xmm4 // Karatsuba pre-processing + movdqu %xmm0,16(%rdi) // save H^2 + palignr $8,%xmm3,%xmm4 // low part is H.lo^H.hi... + movdqu %xmm4,32(%rdi) // save Karatsuba "salt" + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm6,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm6,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // movdqa %xmm0,%xmm5 - movdqa %xmm0,%xmm1 + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm6,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm6,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // pshufd $78,%xmm5,%xmm3 pshufd $78,%xmm0,%xmm4 - pxor %xmm5,%xmm3 - movdqu %xmm5,48(%rdi) - pxor %xmm0,%xmm4 - movdqu %xmm0,64(%rdi) - palignr $8,%xmm3,%xmm4 - movdqu %xmm4,80(%rdi) + pxor %xmm5,%xmm3 // Karatsuba pre-processing + movdqu %xmm5,48(%rdi) // save H^3 + pxor %xmm0,%xmm4 // Karatsuba pre-processing + movdqu %xmm0,64(%rdi) // save H^4 + palignr $8,%xmm3,%xmm4 // low part is H^3.lo^H^3.hi... + movdqu %xmm4,80(%rdi) // save Karatsuba "salt" ret .cfi_endproc @@ -180,43 +180,43 @@ movdqu (%rsi),%xmm2 movdqu 32(%rsi),%xmm4 pshufb %xmm5,%xmm0 - movdqa %xmm0,%xmm1 + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm4,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm4,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // pshufb %xmm5,%xmm0 movdqu %xmm0,(%rdi) ret @@ -246,13 +246,13 @@ jb .Lskip4x subq $0x30,%rcx - movq $0xA040608020C0E000,%rax + movq $0xA040608020C0E000,%rax // ((7..0)·0xE0)&0xff movdqu 48(%rsi),%xmm14 movdqu 64(%rsi),%xmm15 - - - +// ###### +// Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P +// movdqu 48(%rdx),%xmm3 movdqu 32(%rdx),%xmm11 pshufb %xmm10,%xmm3 @@ -317,33 +317,33 @@ pclmulqdq $0x00,%xmm2,%xmm11 pshufd $78,%xmm3,%xmm4 - pxor %xmm0,%xmm8 + pxor %xmm0,%xmm8 // aggregated Karatsuba post-processing movdqa %xmm3,%xmm5 - pxor %xmm1,%xmm8 + pxor %xmm1,%xmm8 // pxor %xmm3,%xmm4 - movdqa %xmm8,%xmm9 + movdqa %xmm8,%xmm9 // pclmulqdq $0x11,%xmm2,%xmm13 pslldq $8,%xmm8 - psrldq $8,%xmm9 + psrldq $8,%xmm9 // pxor %xmm8,%xmm0 movdqa .L7_mask(%rip),%xmm8 - pxor %xmm9,%xmm1 + pxor %xmm9,%xmm1 // movq %rax,%xmm9 - pand %xmm0,%xmm8 - pshufb %xmm8,%xmm9 - pxor %xmm0,%xmm9 + pand %xmm0,%xmm8 // 1st phase + pshufb %xmm8,%xmm9 // + pxor %xmm0,%xmm9 // pclmulqdq $0x00,%xmm7,%xmm12 - psllq $57,%xmm9 - movdqa %xmm9,%xmm8 + psllq $57,%xmm9 // + movdqa %xmm9,%xmm8 // pslldq $8,%xmm9 pclmulqdq $0x00,%xmm6,%xmm3 - psrldq $8,%xmm8 + psrldq $8,%xmm8 // pxor %xmm9,%xmm0 - pxor %xmm8,%xmm1 + pxor %xmm8,%xmm1 // movdqu 0(%rdx),%xmm8 - movdqa %xmm0,%xmm9 + movdqa %xmm0,%xmm9 // 2nd phase psrlq $1,%xmm0 pclmulqdq $0x11,%xmm6,%xmm5 xorps %xmm11,%xmm3 @@ -353,19 +353,19 @@ xorps %xmm13,%xmm5 movups 80(%rsi),%xmm7 pshufb %xmm10,%xmm8 - pxor %xmm9,%xmm1 + pxor %xmm9,%xmm1 // pxor %xmm0,%xmm9 psrlq $5,%xmm0 movdqa %xmm11,%xmm13 pxor %xmm12,%xmm4 pshufd $78,%xmm11,%xmm12 - pxor %xmm9,%xmm0 + pxor %xmm9,%xmm0 // pxor %xmm8,%xmm1 pxor %xmm11,%xmm12 pclmulqdq $0x00,%xmm14,%xmm11 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // movdqa %xmm0,%xmm1 pclmulqdq $0x11,%xmm14,%xmm13 xorps %xmm11,%xmm3 @@ -386,56 +386,56 @@ xorps %xmm12,%xmm4 xorps %xmm3,%xmm0 xorps %xmm5,%xmm1 - pxor %xmm0,%xmm1 + pxor %xmm0,%xmm1 // aggregated Karatsuba post-processing pxor %xmm4,%xmm8 - pxor %xmm1,%xmm8 + pxor %xmm1,%xmm8 // pxor %xmm0,%xmm1 - movdqa %xmm8,%xmm9 + movdqa %xmm8,%xmm9 // psrldq $8,%xmm8 - pslldq $8,%xmm9 + pslldq $8,%xmm9 // pxor %xmm8,%xmm1 - pxor %xmm9,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm9,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // addq $0x40,%rcx jz .Ldone movdqu 32(%rsi),%xmm7 subq $0x10,%rcx jz .Lodd_tail .Lskip4x: - - - - - - movdqu (%rdx),%xmm8 - movdqu 16(%rdx),%xmm3 +// ###### +// Xi+2 =[H*(Ii+1 + Xi+1)] mod P = +// [(H*Ii+1) + (H*Xi+1)] mod P = +// [(H*Ii+1) + H^2*(Ii+Xi)] mod P +// + movdqu (%rdx),%xmm8 // Ii + movdqu 16(%rdx),%xmm3 // Ii+1 pshufb %xmm10,%xmm8 pshufb %xmm10,%xmm3 - pxor %xmm8,%xmm0 + pxor %xmm8,%xmm0 // Ii+Xi movdqa %xmm3,%xmm5 pshufd $78,%xmm3,%xmm4 @@ -444,7 +444,7 @@ pclmulqdq $0x11,%xmm2,%xmm5 pclmulqdq $0x00,%xmm7,%xmm4 - leaq 32(%rdx),%rdx + leaq 32(%rdx),%rdx // i+=2 nop subq $0x20,%rcx jbe .Leven_tail @@ -455,59 +455,59 @@ .Lmod_loop: movdqa %xmm0,%xmm1 movdqa %xmm4,%xmm8 - pshufd $78,%xmm0,%xmm4 - pxor %xmm0,%xmm4 + pshufd $78,%xmm0,%xmm4 // + pxor %xmm0,%xmm4 // pclmulqdq $0x00,%xmm6,%xmm0 pclmulqdq $0x11,%xmm6,%xmm1 pclmulqdq $0x10,%xmm7,%xmm4 - pxor %xmm3,%xmm0 + pxor %xmm3,%xmm0 // (H*Ii+1) + H^2*(Ii+Xi) pxor %xmm5,%xmm1 - movdqu (%rdx),%xmm9 - pxor %xmm0,%xmm8 + movdqu (%rdx),%xmm9 // Ii + pxor %xmm0,%xmm8 // aggregated Karatsuba post-processing pshufb %xmm10,%xmm9 - movdqu 16(%rdx),%xmm3 + movdqu 16(%rdx),%xmm3 // Ii+1 pxor %xmm1,%xmm8 - pxor %xmm9,%xmm1 + pxor %xmm9,%xmm1 // "Ii+Xi", consume early pxor %xmm8,%xmm4 pshufb %xmm10,%xmm3 - movdqa %xmm4,%xmm8 + movdqa %xmm4,%xmm8 // psrldq $8,%xmm8 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm8,%xmm1 - pxor %xmm4,%xmm0 + pxor %xmm4,%xmm0 // - movdqa %xmm3,%xmm5 + movdqa %xmm3,%xmm5 // - movdqa %xmm0,%xmm9 + movdqa %xmm0,%xmm9 // 1st phase movdqa %xmm0,%xmm8 psllq $5,%xmm0 - pxor %xmm0,%xmm8 - pclmulqdq $0x00,%xmm2,%xmm3 + pxor %xmm0,%xmm8 // + pclmulqdq $0x00,%xmm2,%xmm3 // ###### psllq $1,%xmm0 - pxor %xmm8,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm8 + pxor %xmm8,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm8 // pslldq $8,%xmm0 - psrldq $8,%xmm8 + psrldq $8,%xmm8 // pxor %xmm9,%xmm0 pshufd $78,%xmm5,%xmm4 - pxor %xmm8,%xmm1 - pxor %xmm5,%xmm4 + pxor %xmm8,%xmm1 // + pxor %xmm5,%xmm4 // - movdqa %xmm0,%xmm9 + movdqa %xmm0,%xmm9 // 2nd phase psrlq $1,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm5 - pxor %xmm9,%xmm1 + pclmulqdq $0x11,%xmm2,%xmm5 // ###### + pxor %xmm9,%xmm1 // pxor %xmm0,%xmm9 psrlq $5,%xmm0 - pxor %xmm9,%xmm0 + pxor %xmm9,%xmm0 // leaq 32(%rdx),%rdx - psrlq $1,%xmm0 - pclmulqdq $0x00,%xmm7,%xmm4 - pxor %xmm1,%xmm0 + psrlq $1,%xmm0 // + pclmulqdq $0x00,%xmm7,%xmm4 // ###### + pxor %xmm1,%xmm0 // subq $0x20,%rcx ja .Lmod_loop @@ -515,90 +515,90 @@ .Leven_tail: movdqa %xmm0,%xmm1 movdqa %xmm4,%xmm8 - pshufd $78,%xmm0,%xmm4 - pxor %xmm0,%xmm4 + pshufd $78,%xmm0,%xmm4 // + pxor %xmm0,%xmm4 // pclmulqdq $0x00,%xmm6,%xmm0 pclmulqdq $0x11,%xmm6,%xmm1 pclmulqdq $0x10,%xmm7,%xmm4 - pxor %xmm3,%xmm0 + pxor %xmm3,%xmm0 // (H*Ii+1) + H^2*(Ii+Xi) pxor %xmm5,%xmm1 pxor %xmm0,%xmm8 pxor %xmm1,%xmm8 pxor %xmm8,%xmm4 - movdqa %xmm4,%xmm8 + movdqa %xmm4,%xmm8 // psrldq $8,%xmm8 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm8,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // testq %rcx,%rcx jnz .Ldone .Lodd_tail: - movdqu (%rdx),%xmm8 + movdqu (%rdx),%xmm8 // Ii pshufb %xmm10,%xmm8 - pxor %xmm8,%xmm0 - movdqa %xmm0,%xmm1 + pxor %xmm8,%xmm0 // Ii+Xi + movdqa %xmm0,%xmm1 // pshufd $78,%xmm0,%xmm3 - pxor %xmm0,%xmm3 - pclmulqdq $0x00,%xmm2,%xmm0 - pclmulqdq $0x11,%xmm2,%xmm1 - pclmulqdq $0x00,%xmm7,%xmm3 - pxor %xmm0,%xmm3 - pxor %xmm1,%xmm3 + pxor %xmm0,%xmm3 // + pclmulqdq $0x00,%xmm2,%xmm0 // ###### + pclmulqdq $0x11,%xmm2,%xmm1 // ###### + pclmulqdq $0x00,%xmm7,%xmm3 // ###### + pxor %xmm0,%xmm3 // + pxor %xmm1,%xmm3 // - movdqa %xmm3,%xmm4 + movdqa %xmm3,%xmm4 // psrldq $8,%xmm3 - pslldq $8,%xmm4 + pslldq $8,%xmm4 // pxor %xmm3,%xmm1 - pxor %xmm4,%xmm0 - - movdqa %xmm0,%xmm4 + pxor %xmm4,%xmm0 // +// 1st phase + movdqa %xmm0,%xmm4 // movdqa %xmm0,%xmm3 psllq $5,%xmm0 - pxor %xmm0,%xmm3 + pxor %xmm0,%xmm3 // psllq $1,%xmm0 - pxor %xmm3,%xmm0 - psllq $57,%xmm0 - movdqa %xmm0,%xmm3 + pxor %xmm3,%xmm0 // + psllq $57,%xmm0 // + movdqa %xmm0,%xmm3 // pslldq $8,%xmm0 - psrldq $8,%xmm3 + psrldq $8,%xmm3 // pxor %xmm4,%xmm0 - pxor %xmm3,%xmm1 + pxor %xmm3,%xmm1 // - +// 2nd phase movdqa %xmm0,%xmm4 psrlq $1,%xmm0 - pxor %xmm4,%xmm1 + pxor %xmm4,%xmm1 // pxor %xmm0,%xmm4 psrlq $5,%xmm0 - pxor %xmm4,%xmm0 - psrlq $1,%xmm0 - pxor %xmm1,%xmm0 + pxor %xmm4,%xmm0 // + psrlq $1,%xmm0 // + pxor %xmm1,%xmm0 // .Ldone: pshufb %xmm10,%xmm0 movdqu %xmm0,(%rdi) @@ -617,101 +617,101 @@ vzeroupper vmovdqu (%rsi),%xmm2 - vpshufd $78,%xmm2,%xmm2 + vpshufd $78,%xmm2,%xmm2 // dword swap - - vpshufd $255,%xmm2,%xmm4 +// <<1 twist + vpshufd $255,%xmm2,%xmm4 // broadcast uppermost dword vpsrlq $63,%xmm2,%xmm3 vpsllq $1,%xmm2,%xmm2 - vpxor %xmm5,%xmm5,%xmm5 - vpcmpgtd %xmm4,%xmm5,%xmm5 + vpxor %xmm5,%xmm5,%xmm5 // + vpcmpgtd %xmm4,%xmm5,%xmm5 // broadcast carry bit vpslldq $8,%xmm3,%xmm3 - vpor %xmm3,%xmm2,%xmm2 + vpor %xmm3,%xmm2,%xmm2 // H<<=1 - +// magic reduction vpand .L0x1c2_polynomial(%rip),%xmm5,%xmm5 - vpxor %xmm5,%xmm2,%xmm2 + vpxor %xmm5,%xmm2,%xmm2 // if(carry) H^=0x1c2_polynomial vpunpckhqdq %xmm2,%xmm2,%xmm6 vmovdqa %xmm2,%xmm0 vpxor %xmm2,%xmm6,%xmm6 - movq $4,%r10 + movq $4,%r10 // up to H^8 jmp .Linit_start_avx .align 32 .Linit_loop_avx: - vpalignr $8,%xmm3,%xmm4,%xmm5 - vmovdqu %xmm5,-16(%rdi) + vpalignr $8,%xmm3,%xmm4,%xmm5 // low part is H.lo^H.hi... + vmovdqu %xmm5,-16(%rdi) // save Karatsuba "salt" vpunpckhqdq %xmm0,%xmm0,%xmm3 - vpxor %xmm0,%xmm3,%xmm3 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 - vpxor %xmm0,%xmm1,%xmm4 - vpxor %xmm4,%xmm3,%xmm3 + vpxor %xmm0,%xmm3,%xmm3 // + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 // ###### + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 // ###### + vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 // ###### + vpxor %xmm0,%xmm1,%xmm4 // + vpxor %xmm4,%xmm3,%xmm3 // - vpslldq $8,%xmm3,%xmm4 + vpslldq $8,%xmm3,%xmm4 // vpsrldq $8,%xmm3,%xmm3 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpxor %xmm3,%xmm1,%xmm1 - vpsllq $57,%xmm0,%xmm3 + vpsllq $57,%xmm0,%xmm3 // 1st phase vpsllq $62,%xmm0,%xmm4 - vpxor %xmm3,%xmm4,%xmm4 + vpxor %xmm3,%xmm4,%xmm4 // vpsllq $63,%xmm0,%xmm3 - vpxor %xmm3,%xmm4,%xmm4 - vpslldq $8,%xmm4,%xmm3 + vpxor %xmm3,%xmm4,%xmm4 // + vpslldq $8,%xmm4,%xmm3 // vpsrldq $8,%xmm4,%xmm4 - vpxor %xmm3,%xmm0,%xmm0 + vpxor %xmm3,%xmm0,%xmm0 // vpxor %xmm4,%xmm1,%xmm1 - vpsrlq $1,%xmm0,%xmm4 + vpsrlq $1,%xmm0,%xmm4 // 2nd phase vpxor %xmm0,%xmm1,%xmm1 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpsrlq $5,%xmm4,%xmm4 - vpxor %xmm4,%xmm0,%xmm0 - vpsrlq $1,%xmm0,%xmm0 - vpxor %xmm1,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // + vpsrlq $1,%xmm0,%xmm0 // + vpxor %xmm1,%xmm0,%xmm0 // .Linit_start_avx: vmovdqa %xmm0,%xmm5 vpunpckhqdq %xmm0,%xmm0,%xmm3 - vpxor %xmm0,%xmm3,%xmm3 - vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 - vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 - vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 - vpxor %xmm0,%xmm1,%xmm4 - vpxor %xmm4,%xmm3,%xmm3 + vpxor %xmm0,%xmm3,%xmm3 // + vpclmulqdq $0x11,%xmm2,%xmm0,%xmm1 // ###### + vpclmulqdq $0x00,%xmm2,%xmm0,%xmm0 // ###### + vpclmulqdq $0x00,%xmm6,%xmm3,%xmm3 // ###### + vpxor %xmm0,%xmm1,%xmm4 // + vpxor %xmm4,%xmm3,%xmm3 // - vpslldq $8,%xmm3,%xmm4 + vpslldq $8,%xmm3,%xmm4 // vpsrldq $8,%xmm3,%xmm3 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpxor %xmm3,%xmm1,%xmm1 - vpsllq $57,%xmm0,%xmm3 + vpsllq $57,%xmm0,%xmm3 // 1st phase vpsllq $62,%xmm0,%xmm4 - vpxor %xmm3,%xmm4,%xmm4 + vpxor %xmm3,%xmm4,%xmm4 // vpsllq $63,%xmm0,%xmm3 - vpxor %xmm3,%xmm4,%xmm4 - vpslldq $8,%xmm4,%xmm3 + vpxor %xmm3,%xmm4,%xmm4 // + vpslldq $8,%xmm4,%xmm3 // vpsrldq $8,%xmm4,%xmm4 - vpxor %xmm3,%xmm0,%xmm0 + vpxor %xmm3,%xmm0,%xmm0 // vpxor %xmm4,%xmm1,%xmm1 - vpsrlq $1,%xmm0,%xmm4 + vpsrlq $1,%xmm0,%xmm4 // 2nd phase vpxor %xmm0,%xmm1,%xmm1 - vpxor %xmm4,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // vpsrlq $5,%xmm4,%xmm4 - vpxor %xmm4,%xmm0,%xmm0 - vpsrlq $1,%xmm0,%xmm0 - vpxor %xmm1,%xmm0,%xmm0 + vpxor %xmm4,%xmm0,%xmm0 // + vpsrlq $1,%xmm0,%xmm0 // + vpxor %xmm1,%xmm0,%xmm0 // vpshufd $78,%xmm5,%xmm3 vpshufd $78,%xmm0,%xmm4 - vpxor %xmm5,%xmm3,%xmm3 - vmovdqu %xmm5,0(%rdi) - vpxor %xmm0,%xmm4,%xmm4 - vmovdqu %xmm0,16(%rdi) + vpxor %xmm5,%xmm3,%xmm3 // Karatsuba pre-processing + vmovdqu %xmm5,0(%rdi) // save H^1,3,5,7 + vpxor %xmm0,%xmm4,%xmm4 // Karatsuba pre-processing + vmovdqu %xmm0,16(%rdi) // save H^2,4,6,8 leaq 48(%rdi),%rdi subq $1,%r10 jnz .Linit_loop_avx - vpalignr $8,%xmm4,%xmm3,%xmm5 + vpalignr $8,%xmm4,%xmm3,%xmm5 // last "salt" is flipped vmovdqu %xmm5,-16(%rdi) vzeroupper @@ -739,29 +739,29 @@ _CET_ENDBR vzeroupper - vmovdqu (%rdi),%xmm10 + vmovdqu (%rdi),%xmm10 // load %xmm10 leaq .L0x1c2_polynomial(%rip),%r10 - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // size optimization vmovdqu .Lbswap_mask(%rip),%xmm13 vpshufb %xmm13,%xmm10,%xmm10 cmpq $0x80,%rcx jb .Lshort_avx subq $0x80,%rcx - vmovdqu 112(%rdx),%xmm14 - vmovdqu 0-64(%rsi),%xmm6 + vmovdqu 112(%rdx),%xmm14 // I[7] + vmovdqu 0-64(%rsi),%xmm6 // %xmm6^1 vpshufb %xmm13,%xmm14,%xmm14 vmovdqu 32-64(%rsi),%xmm7 vpunpckhqdq %xmm14,%xmm14,%xmm9 - vmovdqu 96(%rdx),%xmm15 + vmovdqu 96(%rdx),%xmm15 // I[6] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpxor %xmm14,%xmm9,%xmm9 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 16-64(%rsi),%xmm6 + vmovdqu 16-64(%rsi),%xmm6 // %xmm6^2 vpunpckhqdq %xmm15,%xmm15,%xmm8 - vmovdqu 80(%rdx),%xmm14 + vmovdqu 80(%rdx),%xmm14 // I[5] vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm15,%xmm8,%xmm8 @@ -769,9 +769,9 @@ vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 48-64(%rsi),%xmm6 + vmovdqu 48-64(%rsi),%xmm6 // %xmm6^3 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu 64(%rdx),%xmm15 + vmovdqu 64(%rdx),%xmm15 // I[4] vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 vmovdqu 80-64(%rsi),%xmm7 @@ -781,56 +781,56 @@ vpxor %xmm1,%xmm4,%xmm4 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 64-64(%rsi),%xmm6 + vmovdqu 64-64(%rsi),%xmm6 // %xmm6^4 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm15,%xmm8,%xmm8 - vmovdqu 48(%rdx),%xmm14 + vmovdqu 48(%rdx),%xmm14 // I[3] vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpxor %xmm4,%xmm1,%xmm1 vpshufb %xmm13,%xmm14,%xmm14 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 96-64(%rsi),%xmm6 + vmovdqu 96-64(%rsi),%xmm6 // %xmm6^5 vpxor %xmm5,%xmm2,%xmm2 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 vmovdqu 128-64(%rsi),%xmm7 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu 32(%rdx),%xmm15 + vmovdqu 32(%rdx),%xmm15 // I[2] vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpxor %xmm1,%xmm4,%xmm4 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 112-64(%rsi),%xmm6 + vmovdqu 112-64(%rsi),%xmm6 // %xmm6^6 vpxor %xmm2,%xmm5,%xmm5 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm15,%xmm8,%xmm8 - vmovdqu 16(%rdx),%xmm14 + vmovdqu 16(%rdx),%xmm14 // I[1] vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpxor %xmm4,%xmm1,%xmm1 vpshufb %xmm13,%xmm14,%xmm14 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 144-64(%rsi),%xmm6 + vmovdqu 144-64(%rsi),%xmm6 // %xmm6^7 vpxor %xmm5,%xmm2,%xmm2 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 vmovdqu 176-64(%rsi),%xmm7 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu (%rdx),%xmm15 + vmovdqu (%rdx),%xmm15 // I[0] vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpxor %xmm1,%xmm4,%xmm4 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 160-64(%rsi),%xmm6 + vmovdqu 160-64(%rsi),%xmm6 // %xmm6^8 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x10,%xmm7,%xmm9,%xmm2 @@ -838,41 +838,41 @@ cmpq $0x80,%rcx jb .Ltail_avx - vpxor %xmm10,%xmm15,%xmm15 + vpxor %xmm10,%xmm15,%xmm15 // accumulate %xmm10 subq $0x80,%rcx jmp .Loop8x_avx .align 32 .Loop8x_avx: vpunpckhqdq %xmm15,%xmm15,%xmm8 - vmovdqu 112(%rdx),%xmm14 + vmovdqu 112(%rdx),%xmm14 // I[7] vpxor %xmm0,%xmm3,%xmm3 vpxor %xmm15,%xmm8,%xmm8 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm10 vpshufb %xmm13,%xmm14,%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm11 - vmovdqu 0-64(%rsi),%xmm6 + vmovdqu 0-64(%rsi),%xmm6 // %xmm6^1 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm12 vmovdqu 32-64(%rsi),%xmm7 vpxor %xmm14,%xmm9,%xmm9 - vmovdqu 96(%rdx),%xmm15 + vmovdqu 96(%rdx),%xmm15 // I[6] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 - vpxor %xmm3,%xmm10,%xmm10 + vpxor %xmm3,%xmm10,%xmm10 // collect result vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 vxorps %xmm4,%xmm11,%xmm11 - vmovdqu 16-64(%rsi),%xmm6 + vmovdqu 16-64(%rsi),%xmm6 // %xmm6^2 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vpxor %xmm5,%xmm12,%xmm12 vxorps %xmm15,%xmm8,%xmm8 - vmovdqu 80(%rdx),%xmm14 - vpxor %xmm10,%xmm12,%xmm12 + vmovdqu 80(%rdx),%xmm14 // I[5] + vpxor %xmm10,%xmm12,%xmm12 // aggregated Karatsuba post-processing vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpxor %xmm11,%xmm12,%xmm12 vpslldq $8,%xmm12,%xmm9 @@ -880,7 +880,7 @@ vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 vpsrldq $8,%xmm12,%xmm12 vpxor %xmm9,%xmm10,%xmm10 - vmovdqu 48-64(%rsi),%xmm6 + vmovdqu 48-64(%rsi),%xmm6 // %xmm6^3 vpshufb %xmm13,%xmm14,%xmm14 vxorps %xmm12,%xmm11,%xmm11 vpxor %xmm1,%xmm4,%xmm4 @@ -890,26 +890,26 @@ vpxor %xmm14,%xmm9,%xmm9 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu 64(%rdx),%xmm15 - vpalignr $8,%xmm10,%xmm10,%xmm12 + vmovdqu 64(%rdx),%xmm15 // I[4] + vpalignr $8,%xmm10,%xmm10,%xmm12 // 1st phase vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpshufb %xmm13,%xmm15,%xmm15 vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 64-64(%rsi),%xmm6 + vmovdqu 64-64(%rsi),%xmm6 // %xmm6^4 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpxor %xmm4,%xmm1,%xmm1 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 vxorps %xmm15,%xmm8,%xmm8 vpxor %xmm5,%xmm2,%xmm2 - vmovdqu 48(%rdx),%xmm14 + vmovdqu 48(%rdx),%xmm14 // I[3] vpclmulqdq $0x10,(%r10),%xmm10,%xmm10 vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpshufb %xmm13,%xmm14,%xmm14 vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 96-64(%rsi),%xmm6 + vmovdqu 96-64(%rsi),%xmm6 // %xmm6^5 vpunpckhqdq %xmm14,%xmm14,%xmm9 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x10,%xmm7,%xmm8,%xmm5 @@ -917,12 +917,12 @@ vpxor %xmm14,%xmm9,%xmm9 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu 32(%rdx),%xmm15 + vmovdqu 32(%rdx),%xmm15 // I[2] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpshufb %xmm13,%xmm15,%xmm15 vpxor %xmm3,%xmm0,%xmm0 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 112-64(%rsi),%xmm6 + vmovdqu 112-64(%rsi),%xmm6 // %xmm6^6 vpunpckhqdq %xmm15,%xmm15,%xmm8 vpxor %xmm4,%xmm1,%xmm1 vpclmulqdq $0x00,%xmm7,%xmm9,%xmm2 @@ -930,13 +930,13 @@ vpxor %xmm5,%xmm2,%xmm2 vxorps %xmm12,%xmm10,%xmm10 - vmovdqu 16(%rdx),%xmm14 - vpalignr $8,%xmm10,%xmm10,%xmm12 + vmovdqu 16(%rdx),%xmm14 // I[1] + vpalignr $8,%xmm10,%xmm10,%xmm12 // 2nd phase vpclmulqdq $0x00,%xmm6,%xmm15,%xmm3 vpshufb %xmm13,%xmm14,%xmm14 vpxor %xmm0,%xmm3,%xmm3 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm4 - vmovdqu 144-64(%rsi),%xmm6 + vmovdqu 144-64(%rsi),%xmm6 // %xmm6^7 vpclmulqdq $0x10,(%r10),%xmm10,%xmm10 vxorps %xmm11,%xmm12,%xmm12 vpunpckhqdq %xmm14,%xmm14,%xmm9 @@ -946,14 +946,14 @@ vpxor %xmm14,%xmm9,%xmm9 vpxor %xmm2,%xmm5,%xmm5 - vmovdqu (%rdx),%xmm15 + vmovdqu (%rdx),%xmm15 // I[0] vpclmulqdq $0x00,%xmm6,%xmm14,%xmm0 vpshufb %xmm13,%xmm15,%xmm15 vpclmulqdq $0x11,%xmm6,%xmm14,%xmm1 - vmovdqu 160-64(%rsi),%xmm6 + vmovdqu 160-64(%rsi),%xmm6 // %xmm6^8 vpxor %xmm12,%xmm15,%xmm15 vpclmulqdq $0x10,%xmm7,%xmm9,%xmm2 - vpxor %xmm10,%xmm15,%xmm15 + vpxor %xmm10,%xmm15,%xmm15 // accumulate %xmm10 leaq 128(%rdx),%rdx subq $0x80,%rcx @@ -964,15 +964,15 @@ .align 32 .Lshort_avx: - vmovdqu -16(%rdx,%rcx,1),%xmm14 + vmovdqu -16(%rdx,%rcx,1),%xmm14 // very last word leaq (%rdx,%rcx,1),%rdx - vmovdqu 0-64(%rsi),%xmm6 + vmovdqu 0-64(%rsi),%xmm6 // %xmm6^1 vmovdqu 32-64(%rsi),%xmm7 vpshufb %xmm13,%xmm14,%xmm15 - vmovdqa %xmm0,%xmm3 - vmovdqa %xmm1,%xmm4 - vmovdqa %xmm2,%xmm5 + vmovdqa %xmm0,%xmm3 // subtle way to zero %xmm3, + vmovdqa %xmm1,%xmm4 // %xmm4 and + vmovdqa %xmm2,%xmm5 // %xmm5 subq $0x10,%rcx jz .Ltail_avx @@ -983,7 +983,7 @@ vmovdqu -32(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 16-64(%rsi),%xmm6 + vmovdqu 16-64(%rsi),%xmm6 // %xmm6^2 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -998,7 +998,7 @@ vmovdqu -48(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 48-64(%rsi),%xmm6 + vmovdqu 48-64(%rsi),%xmm6 // %xmm6^3 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1013,7 +1013,7 @@ vmovdqu -64(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 64-64(%rsi),%xmm6 + vmovdqu 64-64(%rsi),%xmm6 // %xmm6^4 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1028,7 +1028,7 @@ vmovdqu -80(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 96-64(%rsi),%xmm6 + vmovdqu 96-64(%rsi),%xmm6 // %xmm6^5 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1043,7 +1043,7 @@ vmovdqu -96(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 112-64(%rsi),%xmm6 + vmovdqu 112-64(%rsi),%xmm6 // %xmm6^6 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1058,7 +1058,7 @@ vmovdqu -112(%rdx),%xmm14 vpxor %xmm1,%xmm4,%xmm4 vpclmulqdq $0x11,%xmm6,%xmm15,%xmm1 - vmovdqu 144-64(%rsi),%xmm6 + vmovdqu 144-64(%rsi),%xmm6 // %xmm6^7 vpshufb %xmm13,%xmm14,%xmm15 vpxor %xmm2,%xmm5,%xmm5 vpclmulqdq $0x00,%xmm7,%xmm8,%xmm2 @@ -1068,7 +1068,7 @@ .align 32 .Ltail_avx: - vpxor %xmm10,%xmm15,%xmm15 + vpxor %xmm10,%xmm15,%xmm15 // accumulate %xmm10 .Ltail_no_xor_avx: vpunpckhqdq %xmm15,%xmm15,%xmm8 vpxor %xmm0,%xmm3,%xmm3 @@ -1085,18 +1085,18 @@ vpxor %xmm1,%xmm4,%xmm11 vpxor %xmm2,%xmm5,%xmm5 - vpxor %xmm10,%xmm5,%xmm5 + vpxor %xmm10,%xmm5,%xmm5 // aggregated Karatsuba post-processing vpxor %xmm11,%xmm5,%xmm5 vpslldq $8,%xmm5,%xmm9 vpsrldq $8,%xmm5,%xmm5 vpxor %xmm9,%xmm10,%xmm10 vpxor %xmm5,%xmm11,%xmm11 - vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 + vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 // 1st phase vpalignr $8,%xmm10,%xmm10,%xmm10 vpxor %xmm9,%xmm10,%xmm10 - vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 + vpclmulqdq $0x10,%xmm12,%xmm10,%xmm9 // 2nd phase vpalignr $8,%xmm10,%xmm10,%xmm10 vpxor %xmm11,%xmm10,%xmm10 vpxor %xmm9,%xmm10,%xmm10
diff --git a/gen/bcm/ghash-x86_64-win.asm b/gen/bcm/ghash-x86_64-win.asm index a2c71de..eacd900 100644 --- a/gen/bcm/ghash-x86_64-win.asm +++ b/gen/bcm/ghash-x86_64-win.asm
@@ -27,154 +27,154 @@ $L$SEH_prologue_gcm_init_clmul_3: $L$SEH_endprologue_gcm_init_clmul_4: movdqu xmm2,XMMWORD[rdx] - pshufd xmm2,xmm2,78 + pshufd xmm2,xmm2,78 ; dword swap - - pshufd xmm4,xmm2,255 +; <<1 twist + pshufd xmm4,xmm2,255 ; broadcast uppermost dword movdqa xmm3,xmm2 psllq xmm2,1 - pxor xmm5,xmm5 + pxor xmm5,xmm5 ; psrlq xmm3,63 - pcmpgtd xmm5,xmm4 + pcmpgtd xmm5,xmm4 ; broadcast carry bit pslldq xmm3,8 - por xmm2,xmm3 + por xmm2,xmm3 ; H<<=1 - +; magic reduction pand xmm5,XMMWORD[$L$0x1c2_polynomial] - pxor xmm2,xmm5 + pxor xmm2,xmm5 ; if(carry) H^=0x1c2_polynomial - +; calculate H^2 pshufd xmm6,xmm2,78 movdqa xmm0,xmm2 pxor xmm6,xmm2 - movdqa xmm1,xmm0 + movdqa xmm1,xmm0 ; pshufd xmm3,xmm0,78 - pxor xmm3,xmm0 - pclmulqdq xmm0,xmm2,0x00 - pclmulqdq xmm1,xmm2,0x11 - pclmulqdq xmm3,xmm6,0x00 - pxor xmm3,xmm0 - pxor xmm3,xmm1 + pxor xmm3,xmm0 ; + pclmulqdq xmm0,xmm2,0x00 ; ###### + pclmulqdq xmm1,xmm2,0x11 ; ###### + pclmulqdq xmm3,xmm6,0x00 ; ###### + pxor xmm3,xmm0 ; + pxor xmm3,xmm1 ; - movdqa xmm4,xmm3 + movdqa xmm4,xmm3 ; psrldq xmm3,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm3 - pxor xmm0,xmm4 - - movdqa xmm4,xmm0 + pxor xmm0,xmm4 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; pshufd xmm3,xmm2,78 pshufd xmm4,xmm0,78 - pxor xmm3,xmm2 - movdqu XMMWORD[rcx],xmm2 - pxor xmm4,xmm0 - movdqu XMMWORD[16+rcx],xmm0 - palignr xmm4,xmm3,8 - movdqu XMMWORD[32+rcx],xmm4 - movdqa xmm1,xmm0 + pxor xmm3,xmm2 ; Karatsuba pre-processing + movdqu XMMWORD[rcx],xmm2 ; save H + pxor xmm4,xmm0 ; Karatsuba pre-processing + movdqu XMMWORD[16+rcx],xmm0 ; save H^2 + palignr xmm4,xmm3,8 ; low part is H.lo^H.hi... + movdqu XMMWORD[32+rcx],xmm4 ; save Karatsuba "salt" + movdqa xmm1,xmm0 ; pshufd xmm3,xmm0,78 - pxor xmm3,xmm0 - pclmulqdq xmm0,xmm2,0x00 - pclmulqdq xmm1,xmm2,0x11 - pclmulqdq xmm3,xmm6,0x00 - pxor xmm3,xmm0 - pxor xmm3,xmm1 + pxor xmm3,xmm0 ; + pclmulqdq xmm0,xmm2,0x00 ; ###### + pclmulqdq xmm1,xmm2,0x11 ; ###### + pclmulqdq xmm3,xmm6,0x00 ; ###### + pxor xmm3,xmm0 ; + pxor xmm3,xmm1 ; - movdqa xmm4,xmm3 + movdqa xmm4,xmm3 ; psrldq xmm3,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm3 - pxor xmm0,xmm4 - - movdqa xmm4,xmm0 + pxor xmm0,xmm4 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; movdqa xmm5,xmm0 - movdqa xmm1,xmm0 + movdqa xmm1,xmm0 ; pshufd xmm3,xmm0,78 - pxor xmm3,xmm0 - pclmulqdq xmm0,xmm2,0x00 - pclmulqdq xmm1,xmm2,0x11 - pclmulqdq xmm3,xmm6,0x00 - pxor xmm3,xmm0 - pxor xmm3,xmm1 + pxor xmm3,xmm0 ; + pclmulqdq xmm0,xmm2,0x00 ; ###### + pclmulqdq xmm1,xmm2,0x11 ; ###### + pclmulqdq xmm3,xmm6,0x00 ; ###### + pxor xmm3,xmm0 ; + pxor xmm3,xmm1 ; - movdqa xmm4,xmm3 + movdqa xmm4,xmm3 ; psrldq xmm3,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm3 - pxor xmm0,xmm4 - - movdqa xmm4,xmm0 + pxor xmm0,xmm4 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; pshufd xmm3,xmm5,78 pshufd xmm4,xmm0,78 - pxor xmm3,xmm5 - movdqu XMMWORD[48+rcx],xmm5 - pxor xmm4,xmm0 - movdqu XMMWORD[64+rcx],xmm0 - palignr xmm4,xmm3,8 - movdqu XMMWORD[80+rcx],xmm4 + pxor xmm3,xmm5 ; Karatsuba pre-processing + movdqu XMMWORD[48+rcx],xmm5 ; save H^3 + pxor xmm4,xmm0 ; Karatsuba pre-processing + movdqu XMMWORD[64+rcx],xmm0 ; save H^4 + palignr xmm4,xmm3,8 ; low part is H^3.lo^H^3.hi... + movdqu XMMWORD[80+rcx],xmm4 ; save Karatsuba "salt" movaps xmm6,XMMWORD[rsp] lea rsp,[24+rsp] ret @@ -193,43 +193,43 @@ movdqu xmm2,XMMWORD[rdx] movdqu xmm4,XMMWORD[32+rdx] pshufb xmm0,xmm5 - movdqa xmm1,xmm0 + movdqa xmm1,xmm0 ; pshufd xmm3,xmm0,78 - pxor xmm3,xmm0 - pclmulqdq xmm0,xmm2,0x00 - pclmulqdq xmm1,xmm2,0x11 - pclmulqdq xmm3,xmm4,0x00 - pxor xmm3,xmm0 - pxor xmm3,xmm1 + pxor xmm3,xmm0 ; + pclmulqdq xmm0,xmm2,0x00 ; ###### + pclmulqdq xmm1,xmm2,0x11 ; ###### + pclmulqdq xmm3,xmm4,0x00 ; ###### + pxor xmm3,xmm0 ; + pxor xmm3,xmm1 ; - movdqa xmm4,xmm3 + movdqa xmm4,xmm3 ; psrldq xmm3,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm3 - pxor xmm0,xmm4 - - movdqa xmm4,xmm0 + pxor xmm0,xmm4 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; pshufb xmm0,xmm5 movdqu XMMWORD[rcx],xmm0 ret @@ -282,13 +282,13 @@ jb NEAR $L$skip4x sub r9,0x30 - mov rax,0xA040608020C0E000 + mov rax,0xA040608020C0E000 ; ((7..0)·0xE0)&0xff movdqu xmm14,XMMWORD[48+rdx] movdqu xmm15,XMMWORD[64+rdx] - - - +; ###### +; Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P +; movdqu xmm3,XMMWORD[48+r8] movdqu xmm11,XMMWORD[32+r8] pshufb xmm3,xmm10 @@ -353,33 +353,33 @@ pclmulqdq xmm11,xmm2,0x00 pshufd xmm4,xmm3,78 - pxor xmm8,xmm0 + pxor xmm8,xmm0 ; aggregated Karatsuba post-processing movdqa xmm5,xmm3 - pxor xmm8,xmm1 + pxor xmm8,xmm1 ; pxor xmm4,xmm3 - movdqa xmm9,xmm8 + movdqa xmm9,xmm8 ; pclmulqdq xmm13,xmm2,0x11 pslldq xmm8,8 - psrldq xmm9,8 + psrldq xmm9,8 ; pxor xmm0,xmm8 movdqa xmm8,XMMWORD[$L$7_mask] - pxor xmm1,xmm9 + pxor xmm1,xmm9 ; movq xmm9,rax - pand xmm8,xmm0 - pshufb xmm9,xmm8 - pxor xmm9,xmm0 + pand xmm8,xmm0 ; 1st phase + pshufb xmm9,xmm8 ; + pxor xmm9,xmm0 ; pclmulqdq xmm12,xmm7,0x00 - psllq xmm9,57 - movdqa xmm8,xmm9 + psllq xmm9,57 ; + movdqa xmm8,xmm9 ; pslldq xmm9,8 pclmulqdq xmm3,xmm6,0x00 - psrldq xmm8,8 + psrldq xmm8,8 ; pxor xmm0,xmm9 - pxor xmm1,xmm8 + pxor xmm1,xmm8 ; movdqu xmm8,XMMWORD[r8] - movdqa xmm9,xmm0 + movdqa xmm9,xmm0 ; 2nd phase psrlq xmm0,1 pclmulqdq xmm5,xmm6,0x11 xorps xmm3,xmm11 @@ -389,19 +389,19 @@ xorps xmm5,xmm13 movups xmm7,XMMWORD[80+rdx] pshufb xmm8,xmm10 - pxor xmm1,xmm9 + pxor xmm1,xmm9 ; pxor xmm9,xmm0 psrlq xmm0,5 movdqa xmm13,xmm11 pxor xmm4,xmm12 pshufd xmm12,xmm11,78 - pxor xmm0,xmm9 + pxor xmm0,xmm9 ; pxor xmm1,xmm8 pxor xmm12,xmm11 pclmulqdq xmm11,xmm14,0x00 - psrlq xmm0,1 - pxor xmm0,xmm1 + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; movdqa xmm1,xmm0 pclmulqdq xmm13,xmm14,0x11 xorps xmm3,xmm11 @@ -422,56 +422,56 @@ xorps xmm4,xmm12 xorps xmm0,xmm3 xorps xmm1,xmm5 - pxor xmm1,xmm0 + pxor xmm1,xmm0 ; aggregated Karatsuba post-processing pxor xmm8,xmm4 - pxor xmm8,xmm1 + pxor xmm8,xmm1 ; pxor xmm1,xmm0 - movdqa xmm9,xmm8 + movdqa xmm9,xmm8 ; psrldq xmm8,8 - pslldq xmm9,8 + pslldq xmm9,8 ; pxor xmm1,xmm8 - pxor xmm0,xmm9 - - movdqa xmm4,xmm0 + pxor xmm0,xmm9 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; add r9,0x40 jz NEAR $L$done movdqu xmm7,XMMWORD[32+rdx] sub r9,0x10 jz NEAR $L$odd_tail $L$skip4x: - - - - - - movdqu xmm8,XMMWORD[r8] - movdqu xmm3,XMMWORD[16+r8] +; ###### +; Xi+2 =[H*(Ii+1 + Xi+1)] mod P = +; [(H*Ii+1) + (H*Xi+1)] mod P = +; [(H*Ii+1) + H^2*(Ii+Xi)] mod P +; + movdqu xmm8,XMMWORD[r8] ; Ii + movdqu xmm3,XMMWORD[16+r8] ; Ii+1 pshufb xmm8,xmm10 pshufb xmm3,xmm10 - pxor xmm0,xmm8 + pxor xmm0,xmm8 ; Ii+Xi movdqa xmm5,xmm3 pshufd xmm4,xmm3,78 @@ -480,7 +480,7 @@ pclmulqdq xmm5,xmm2,0x11 pclmulqdq xmm4,xmm7,0x00 - lea r8,[32+r8] + lea r8,[32+r8] ; i+=2 nop sub r9,0x20 jbe NEAR $L$even_tail @@ -491,59 +491,59 @@ $L$mod_loop: movdqa xmm1,xmm0 movdqa xmm8,xmm4 - pshufd xmm4,xmm0,78 - pxor xmm4,xmm0 + pshufd xmm4,xmm0,78 ; + pxor xmm4,xmm0 ; pclmulqdq xmm0,xmm6,0x00 pclmulqdq xmm1,xmm6,0x11 pclmulqdq xmm4,xmm7,0x10 - pxor xmm0,xmm3 + pxor xmm0,xmm3 ; (H*Ii+1) + H^2*(Ii+Xi) pxor xmm1,xmm5 - movdqu xmm9,XMMWORD[r8] - pxor xmm8,xmm0 + movdqu xmm9,XMMWORD[r8] ; Ii + pxor xmm8,xmm0 ; aggregated Karatsuba post-processing pshufb xmm9,xmm10 - movdqu xmm3,XMMWORD[16+r8] + movdqu xmm3,XMMWORD[16+r8] ; Ii+1 pxor xmm8,xmm1 - pxor xmm1,xmm9 + pxor xmm1,xmm9 ; "Ii+Xi", consume early pxor xmm4,xmm8 pshufb xmm3,xmm10 - movdqa xmm8,xmm4 + movdqa xmm8,xmm4 ; psrldq xmm8,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm8 - pxor xmm0,xmm4 + pxor xmm0,xmm4 ; - movdqa xmm5,xmm3 + movdqa xmm5,xmm3 ; - movdqa xmm9,xmm0 + movdqa xmm9,xmm0 ; 1st phase movdqa xmm8,xmm0 psllq xmm0,5 - pxor xmm8,xmm0 - pclmulqdq xmm3,xmm2,0x00 + pxor xmm8,xmm0 ; + pclmulqdq xmm3,xmm2,0x00 ; ###### psllq xmm0,1 - pxor xmm0,xmm8 - psllq xmm0,57 - movdqa xmm8,xmm0 + pxor xmm0,xmm8 ; + psllq xmm0,57 ; + movdqa xmm8,xmm0 ; pslldq xmm0,8 - psrldq xmm8,8 + psrldq xmm8,8 ; pxor xmm0,xmm9 pshufd xmm4,xmm5,78 - pxor xmm1,xmm8 - pxor xmm4,xmm5 + pxor xmm1,xmm8 ; + pxor xmm4,xmm5 ; - movdqa xmm9,xmm0 + movdqa xmm9,xmm0 ; 2nd phase psrlq xmm0,1 - pclmulqdq xmm5,xmm2,0x11 - pxor xmm1,xmm9 + pclmulqdq xmm5,xmm2,0x11 ; ###### + pxor xmm1,xmm9 ; pxor xmm9,xmm0 psrlq xmm0,5 - pxor xmm0,xmm9 + pxor xmm0,xmm9 ; lea r8,[32+r8] - psrlq xmm0,1 - pclmulqdq xmm4,xmm7,0x00 - pxor xmm0,xmm1 + psrlq xmm0,1 ; + pclmulqdq xmm4,xmm7,0x00 ; ###### + pxor xmm0,xmm1 ; sub r9,0x20 ja NEAR $L$mod_loop @@ -551,90 +551,90 @@ $L$even_tail: movdqa xmm1,xmm0 movdqa xmm8,xmm4 - pshufd xmm4,xmm0,78 - pxor xmm4,xmm0 + pshufd xmm4,xmm0,78 ; + pxor xmm4,xmm0 ; pclmulqdq xmm0,xmm6,0x00 pclmulqdq xmm1,xmm6,0x11 pclmulqdq xmm4,xmm7,0x10 - pxor xmm0,xmm3 + pxor xmm0,xmm3 ; (H*Ii+1) + H^2*(Ii+Xi) pxor xmm1,xmm5 pxor xmm8,xmm0 pxor xmm8,xmm1 pxor xmm4,xmm8 - movdqa xmm8,xmm4 + movdqa xmm8,xmm4 ; psrldq xmm8,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm8 - pxor xmm0,xmm4 - - movdqa xmm4,xmm0 + pxor xmm0,xmm4 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; test r9,r9 jnz NEAR $L$done $L$odd_tail: - movdqu xmm8,XMMWORD[r8] + movdqu xmm8,XMMWORD[r8] ; Ii pshufb xmm8,xmm10 - pxor xmm0,xmm8 - movdqa xmm1,xmm0 + pxor xmm0,xmm8 ; Ii+Xi + movdqa xmm1,xmm0 ; pshufd xmm3,xmm0,78 - pxor xmm3,xmm0 - pclmulqdq xmm0,xmm2,0x00 - pclmulqdq xmm1,xmm2,0x11 - pclmulqdq xmm3,xmm7,0x00 - pxor xmm3,xmm0 - pxor xmm3,xmm1 + pxor xmm3,xmm0 ; + pclmulqdq xmm0,xmm2,0x00 ; ###### + pclmulqdq xmm1,xmm2,0x11 ; ###### + pclmulqdq xmm3,xmm7,0x00 ; ###### + pxor xmm3,xmm0 ; + pxor xmm3,xmm1 ; - movdqa xmm4,xmm3 + movdqa xmm4,xmm3 ; psrldq xmm3,8 - pslldq xmm4,8 + pslldq xmm4,8 ; pxor xmm1,xmm3 - pxor xmm0,xmm4 - - movdqa xmm4,xmm0 + pxor xmm0,xmm4 ; +; 1st phase + movdqa xmm4,xmm0 ; movdqa xmm3,xmm0 psllq xmm0,5 - pxor xmm3,xmm0 + pxor xmm3,xmm0 ; psllq xmm0,1 - pxor xmm0,xmm3 - psllq xmm0,57 - movdqa xmm3,xmm0 + pxor xmm0,xmm3 ; + psllq xmm0,57 ; + movdqa xmm3,xmm0 ; pslldq xmm0,8 - psrldq xmm3,8 + psrldq xmm3,8 ; pxor xmm0,xmm4 - pxor xmm1,xmm3 + pxor xmm1,xmm3 ; - +; 2nd phase movdqa xmm4,xmm0 psrlq xmm0,1 - pxor xmm1,xmm4 + pxor xmm1,xmm4 ; pxor xmm4,xmm0 psrlq xmm0,5 - pxor xmm0,xmm4 - psrlq xmm0,1 - pxor xmm0,xmm1 + pxor xmm0,xmm4 ; + psrlq xmm0,1 ; + pxor xmm0,xmm1 ; $L$done: pshufb xmm0,xmm10 movdqu XMMWORD[rcx],xmm0 @@ -668,101 +668,101 @@ vzeroupper vmovdqu xmm2,XMMWORD[rdx] - vpshufd xmm2,xmm2,78 + vpshufd xmm2,xmm2,78 ; dword swap - - vpshufd xmm4,xmm2,255 +; <<1 twist + vpshufd xmm4,xmm2,255 ; broadcast uppermost dword vpsrlq xmm3,xmm2,63 vpsllq xmm2,xmm2,1 - vpxor xmm5,xmm5,xmm5 - vpcmpgtd xmm5,xmm5,xmm4 + vpxor xmm5,xmm5,xmm5 ; + vpcmpgtd xmm5,xmm5,xmm4 ; broadcast carry bit vpslldq xmm3,xmm3,8 - vpor xmm2,xmm2,xmm3 + vpor xmm2,xmm2,xmm3 ; H<<=1 - +; magic reduction vpand xmm5,xmm5,XMMWORD[$L$0x1c2_polynomial] - vpxor xmm2,xmm2,xmm5 + vpxor xmm2,xmm2,xmm5 ; if(carry) H^=0x1c2_polynomial vpunpckhqdq xmm6,xmm2,xmm2 vmovdqa xmm0,xmm2 vpxor xmm6,xmm6,xmm2 - mov r10,4 + mov r10,4 ; up to H^8 jmp NEAR $L$init_start_avx ALIGN 32 $L$init_loop_avx: - vpalignr xmm5,xmm4,xmm3,8 - vmovdqu XMMWORD[(-16)+rcx],xmm5 + vpalignr xmm5,xmm4,xmm3,8 ; low part is H.lo^H.hi... + vmovdqu XMMWORD[(-16)+rcx],xmm5 ; save Karatsuba "salt" vpunpckhqdq xmm3,xmm0,xmm0 - vpxor xmm3,xmm3,xmm0 - vpclmulqdq xmm1,xmm0,xmm2,0x11 - vpclmulqdq xmm0,xmm0,xmm2,0x00 - vpclmulqdq xmm3,xmm3,xmm6,0x00 - vpxor xmm4,xmm1,xmm0 - vpxor xmm3,xmm3,xmm4 + vpxor xmm3,xmm3,xmm0 ; + vpclmulqdq xmm1,xmm0,xmm2,0x11 ; ###### + vpclmulqdq xmm0,xmm0,xmm2,0x00 ; ###### + vpclmulqdq xmm3,xmm3,xmm6,0x00 ; ###### + vpxor xmm4,xmm1,xmm0 ; + vpxor xmm3,xmm3,xmm4 ; - vpslldq xmm4,xmm3,8 + vpslldq xmm4,xmm3,8 ; vpsrldq xmm3,xmm3,8 - vpxor xmm0,xmm0,xmm4 + vpxor xmm0,xmm0,xmm4 ; vpxor xmm1,xmm1,xmm3 - vpsllq xmm3,xmm0,57 + vpsllq xmm3,xmm0,57 ; 1st phase vpsllq xmm4,xmm0,62 - vpxor xmm4,xmm4,xmm3 + vpxor xmm4,xmm4,xmm3 ; vpsllq xmm3,xmm0,63 - vpxor xmm4,xmm4,xmm3 - vpslldq xmm3,xmm4,8 + vpxor xmm4,xmm4,xmm3 ; + vpslldq xmm3,xmm4,8 ; vpsrldq xmm4,xmm4,8 - vpxor xmm0,xmm0,xmm3 + vpxor xmm0,xmm0,xmm3 ; vpxor xmm1,xmm1,xmm4 - vpsrlq xmm4,xmm0,1 + vpsrlq xmm4,xmm0,1 ; 2nd phase vpxor xmm1,xmm1,xmm0 - vpxor xmm0,xmm0,xmm4 + vpxor xmm0,xmm0,xmm4 ; vpsrlq xmm4,xmm4,5 - vpxor xmm0,xmm0,xmm4 - vpsrlq xmm0,xmm0,1 - vpxor xmm0,xmm0,xmm1 + vpxor xmm0,xmm0,xmm4 ; + vpsrlq xmm0,xmm0,1 ; + vpxor xmm0,xmm0,xmm1 ; $L$init_start_avx: vmovdqa xmm5,xmm0 vpunpckhqdq xmm3,xmm0,xmm0 - vpxor xmm3,xmm3,xmm0 - vpclmulqdq xmm1,xmm0,xmm2,0x11 - vpclmulqdq xmm0,xmm0,xmm2,0x00 - vpclmulqdq xmm3,xmm3,xmm6,0x00 - vpxor xmm4,xmm1,xmm0 - vpxor xmm3,xmm3,xmm4 + vpxor xmm3,xmm3,xmm0 ; + vpclmulqdq xmm1,xmm0,xmm2,0x11 ; ###### + vpclmulqdq xmm0,xmm0,xmm2,0x00 ; ###### + vpclmulqdq xmm3,xmm3,xmm6,0x00 ; ###### + vpxor xmm4,xmm1,xmm0 ; + vpxor xmm3,xmm3,xmm4 ; - vpslldq xmm4,xmm3,8 + vpslldq xmm4,xmm3,8 ; vpsrldq xmm3,xmm3,8 - vpxor xmm0,xmm0,xmm4 + vpxor xmm0,xmm0,xmm4 ; vpxor xmm1,xmm1,xmm3 - vpsllq xmm3,xmm0,57 + vpsllq xmm3,xmm0,57 ; 1st phase vpsllq xmm4,xmm0,62 - vpxor xmm4,xmm4,xmm3 + vpxor xmm4,xmm4,xmm3 ; vpsllq xmm3,xmm0,63 - vpxor xmm4,xmm4,xmm3 - vpslldq xmm3,xmm4,8 + vpxor xmm4,xmm4,xmm3 ; + vpslldq xmm3,xmm4,8 ; vpsrldq xmm4,xmm4,8 - vpxor xmm0,xmm0,xmm3 + vpxor xmm0,xmm0,xmm3 ; vpxor xmm1,xmm1,xmm4 - vpsrlq xmm4,xmm0,1 + vpsrlq xmm4,xmm0,1 ; 2nd phase vpxor xmm1,xmm1,xmm0 - vpxor xmm0,xmm0,xmm4 + vpxor xmm0,xmm0,xmm4 ; vpsrlq xmm4,xmm4,5 - vpxor xmm0,xmm0,xmm4 - vpsrlq xmm0,xmm0,1 - vpxor xmm0,xmm0,xmm1 + vpxor xmm0,xmm0,xmm4 ; + vpsrlq xmm0,xmm0,1 ; + vpxor xmm0,xmm0,xmm1 ; vpshufd xmm3,xmm5,78 vpshufd xmm4,xmm0,78 - vpxor xmm3,xmm3,xmm5 - vmovdqu XMMWORD[rcx],xmm5 - vpxor xmm4,xmm4,xmm0 - vmovdqu XMMWORD[16+rcx],xmm0 + vpxor xmm3,xmm3,xmm5 ; Karatsuba pre-processing + vmovdqu XMMWORD[rcx],xmm5 ; save H^1,3,5,7 + vpxor xmm4,xmm4,xmm0 ; Karatsuba pre-processing + vmovdqu XMMWORD[16+rcx],xmm0 ; save H^2,4,6,8 lea rcx,[48+rcx] sub r10,1 jnz NEAR $L$init_loop_avx - vpalignr xmm5,xmm3,xmm4,8 + vpalignr xmm5,xmm3,xmm4,8 ; last "salt" is flipped vmovdqu XMMWORD[(-16)+rcx],xmm5 vzeroupper @@ -814,29 +814,29 @@ $L$SEH_endprologue_gcm_ghash_avx_13: vzeroupper - vmovdqu xmm10,XMMWORD[rcx] + vmovdqu xmm10,XMMWORD[rcx] ; load %xmm10 lea r10,[$L$0x1c2_polynomial] - lea rdx,[64+rdx] + lea rdx,[64+rdx] ; size optimization vmovdqu xmm13,XMMWORD[$L$bswap_mask] vpshufb xmm10,xmm10,xmm13 cmp r9,0x80 jb NEAR $L$short_avx sub r9,0x80 - vmovdqu xmm14,XMMWORD[112+r8] - vmovdqu xmm6,XMMWORD[((0-64))+rdx] + vmovdqu xmm14,XMMWORD[112+r8] ; I[7] + vmovdqu xmm6,XMMWORD[((0-64))+rdx] ; %xmm6^1 vpshufb xmm14,xmm14,xmm13 vmovdqu xmm7,XMMWORD[((32-64))+rdx] vpunpckhqdq xmm9,xmm14,xmm14 - vmovdqu xmm15,XMMWORD[96+r8] + vmovdqu xmm15,XMMWORD[96+r8] ; I[6] vpclmulqdq xmm0,xmm14,xmm6,0x00 vpxor xmm9,xmm9,xmm14 vpshufb xmm15,xmm15,xmm13 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((16-64))+rdx] + vmovdqu xmm6,XMMWORD[((16-64))+rdx] ; %xmm6^2 vpunpckhqdq xmm8,xmm15,xmm15 - vmovdqu xmm14,XMMWORD[80+r8] + vmovdqu xmm14,XMMWORD[80+r8] ; I[5] vpclmulqdq xmm2,xmm9,xmm7,0x00 vpxor xmm8,xmm8,xmm15 @@ -844,9 +844,9 @@ vpclmulqdq xmm3,xmm15,xmm6,0x00 vpunpckhqdq xmm9,xmm14,xmm14 vpclmulqdq xmm4,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((48-64))+rdx] + vmovdqu xmm6,XMMWORD[((48-64))+rdx] ; %xmm6^3 vpxor xmm9,xmm9,xmm14 - vmovdqu xmm15,XMMWORD[64+r8] + vmovdqu xmm15,XMMWORD[64+r8] ; I[4] vpclmulqdq xmm5,xmm8,xmm7,0x10 vmovdqu xmm7,XMMWORD[((80-64))+rdx] @@ -856,56 +856,56 @@ vpxor xmm4,xmm4,xmm1 vpunpckhqdq xmm8,xmm15,xmm15 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((64-64))+rdx] + vmovdqu xmm6,XMMWORD[((64-64))+rdx] ; %xmm6^4 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm9,xmm7,0x00 vpxor xmm8,xmm8,xmm15 - vmovdqu xmm14,XMMWORD[48+r8] + vmovdqu xmm14,XMMWORD[48+r8] ; I[3] vpxor xmm0,xmm0,xmm3 vpclmulqdq xmm3,xmm15,xmm6,0x00 vpxor xmm1,xmm1,xmm4 vpshufb xmm14,xmm14,xmm13 vpclmulqdq xmm4,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((96-64))+rdx] + vmovdqu xmm6,XMMWORD[((96-64))+rdx] ; %xmm6^5 vpxor xmm2,xmm2,xmm5 vpunpckhqdq xmm9,xmm14,xmm14 vpclmulqdq xmm5,xmm8,xmm7,0x10 vmovdqu xmm7,XMMWORD[((128-64))+rdx] vpxor xmm9,xmm9,xmm14 - vmovdqu xmm15,XMMWORD[32+r8] + vmovdqu xmm15,XMMWORD[32+r8] ; I[2] vpxor xmm3,xmm3,xmm0 vpclmulqdq xmm0,xmm14,xmm6,0x00 vpxor xmm4,xmm4,xmm1 vpshufb xmm15,xmm15,xmm13 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((112-64))+rdx] + vmovdqu xmm6,XMMWORD[((112-64))+rdx] ; %xmm6^6 vpxor xmm5,xmm5,xmm2 vpunpckhqdq xmm8,xmm15,xmm15 vpclmulqdq xmm2,xmm9,xmm7,0x00 vpxor xmm8,xmm8,xmm15 - vmovdqu xmm14,XMMWORD[16+r8] + vmovdqu xmm14,XMMWORD[16+r8] ; I[1] vpxor xmm0,xmm0,xmm3 vpclmulqdq xmm3,xmm15,xmm6,0x00 vpxor xmm1,xmm1,xmm4 vpshufb xmm14,xmm14,xmm13 vpclmulqdq xmm4,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((144-64))+rdx] + vmovdqu xmm6,XMMWORD[((144-64))+rdx] ; %xmm6^7 vpxor xmm2,xmm2,xmm5 vpunpckhqdq xmm9,xmm14,xmm14 vpclmulqdq xmm5,xmm8,xmm7,0x10 vmovdqu xmm7,XMMWORD[((176-64))+rdx] vpxor xmm9,xmm9,xmm14 - vmovdqu xmm15,XMMWORD[r8] + vmovdqu xmm15,XMMWORD[r8] ; I[0] vpxor xmm3,xmm3,xmm0 vpclmulqdq xmm0,xmm14,xmm6,0x00 vpxor xmm4,xmm4,xmm1 vpshufb xmm15,xmm15,xmm13 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((160-64))+rdx] + vmovdqu xmm6,XMMWORD[((160-64))+rdx] ; %xmm6^8 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm9,xmm7,0x10 @@ -913,41 +913,41 @@ cmp r9,0x80 jb NEAR $L$tail_avx - vpxor xmm15,xmm15,xmm10 + vpxor xmm15,xmm15,xmm10 ; accumulate %xmm10 sub r9,0x80 jmp NEAR $L$oop8x_avx ALIGN 32 $L$oop8x_avx: vpunpckhqdq xmm8,xmm15,xmm15 - vmovdqu xmm14,XMMWORD[112+r8] + vmovdqu xmm14,XMMWORD[112+r8] ; I[7] vpxor xmm3,xmm3,xmm0 vpxor xmm8,xmm8,xmm15 vpclmulqdq xmm10,xmm15,xmm6,0x00 vpshufb xmm14,xmm14,xmm13 vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm11,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((0-64))+rdx] + vmovdqu xmm6,XMMWORD[((0-64))+rdx] ; %xmm6^1 vpunpckhqdq xmm9,xmm14,xmm14 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm12,xmm8,xmm7,0x00 vmovdqu xmm7,XMMWORD[((32-64))+rdx] vpxor xmm9,xmm9,xmm14 - vmovdqu xmm15,XMMWORD[96+r8] + vmovdqu xmm15,XMMWORD[96+r8] ; I[6] vpclmulqdq xmm0,xmm14,xmm6,0x00 - vpxor xmm10,xmm10,xmm3 + vpxor xmm10,xmm10,xmm3 ; collect result vpshufb xmm15,xmm15,xmm13 vpclmulqdq xmm1,xmm14,xmm6,0x11 vxorps xmm11,xmm11,xmm4 - vmovdqu xmm6,XMMWORD[((16-64))+rdx] + vmovdqu xmm6,XMMWORD[((16-64))+rdx] ; %xmm6^2 vpunpckhqdq xmm8,xmm15,xmm15 vpclmulqdq xmm2,xmm9,xmm7,0x00 vpxor xmm12,xmm12,xmm5 vxorps xmm8,xmm8,xmm15 - vmovdqu xmm14,XMMWORD[80+r8] - vpxor xmm12,xmm12,xmm10 + vmovdqu xmm14,XMMWORD[80+r8] ; I[5] + vpxor xmm12,xmm12,xmm10 ; aggregated Karatsuba post-processing vpclmulqdq xmm3,xmm15,xmm6,0x00 vpxor xmm12,xmm12,xmm11 vpslldq xmm9,xmm12,8 @@ -955,7 +955,7 @@ vpclmulqdq xmm4,xmm15,xmm6,0x11 vpsrldq xmm12,xmm12,8 vpxor xmm10,xmm10,xmm9 - vmovdqu xmm6,XMMWORD[((48-64))+rdx] + vmovdqu xmm6,XMMWORD[((48-64))+rdx] ; %xmm6^3 vpshufb xmm14,xmm14,xmm13 vxorps xmm11,xmm11,xmm12 vpxor xmm4,xmm4,xmm1 @@ -965,26 +965,26 @@ vpxor xmm9,xmm9,xmm14 vpxor xmm5,xmm5,xmm2 - vmovdqu xmm15,XMMWORD[64+r8] - vpalignr xmm12,xmm10,xmm10,8 + vmovdqu xmm15,XMMWORD[64+r8] ; I[4] + vpalignr xmm12,xmm10,xmm10,8 ; 1st phase vpclmulqdq xmm0,xmm14,xmm6,0x00 vpshufb xmm15,xmm15,xmm13 vpxor xmm0,xmm0,xmm3 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((64-64))+rdx] + vmovdqu xmm6,XMMWORD[((64-64))+rdx] ; %xmm6^4 vpunpckhqdq xmm8,xmm15,xmm15 vpxor xmm1,xmm1,xmm4 vpclmulqdq xmm2,xmm9,xmm7,0x00 vxorps xmm8,xmm8,xmm15 vpxor xmm2,xmm2,xmm5 - vmovdqu xmm14,XMMWORD[48+r8] + vmovdqu xmm14,XMMWORD[48+r8] ; I[3] vpclmulqdq xmm10,xmm10,XMMWORD[r10],0x10 vpclmulqdq xmm3,xmm15,xmm6,0x00 vpshufb xmm14,xmm14,xmm13 vpxor xmm3,xmm3,xmm0 vpclmulqdq xmm4,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((96-64))+rdx] + vmovdqu xmm6,XMMWORD[((96-64))+rdx] ; %xmm6^5 vpunpckhqdq xmm9,xmm14,xmm14 vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm5,xmm8,xmm7,0x10 @@ -992,12 +992,12 @@ vpxor xmm9,xmm9,xmm14 vpxor xmm5,xmm5,xmm2 - vmovdqu xmm15,XMMWORD[32+r8] + vmovdqu xmm15,XMMWORD[32+r8] ; I[2] vpclmulqdq xmm0,xmm14,xmm6,0x00 vpshufb xmm15,xmm15,xmm13 vpxor xmm0,xmm0,xmm3 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((112-64))+rdx] + vmovdqu xmm6,XMMWORD[((112-64))+rdx] ; %xmm6^6 vpunpckhqdq xmm8,xmm15,xmm15 vpxor xmm1,xmm1,xmm4 vpclmulqdq xmm2,xmm9,xmm7,0x00 @@ -1005,13 +1005,13 @@ vpxor xmm2,xmm2,xmm5 vxorps xmm10,xmm10,xmm12 - vmovdqu xmm14,XMMWORD[16+r8] - vpalignr xmm12,xmm10,xmm10,8 + vmovdqu xmm14,XMMWORD[16+r8] ; I[1] + vpalignr xmm12,xmm10,xmm10,8 ; 2nd phase vpclmulqdq xmm3,xmm15,xmm6,0x00 vpshufb xmm14,xmm14,xmm13 vpxor xmm3,xmm3,xmm0 vpclmulqdq xmm4,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((144-64))+rdx] + vmovdqu xmm6,XMMWORD[((144-64))+rdx] ; %xmm6^7 vpclmulqdq xmm10,xmm10,XMMWORD[r10],0x10 vxorps xmm12,xmm12,xmm11 vpunpckhqdq xmm9,xmm14,xmm14 @@ -1021,14 +1021,14 @@ vpxor xmm9,xmm9,xmm14 vpxor xmm5,xmm5,xmm2 - vmovdqu xmm15,XMMWORD[r8] + vmovdqu xmm15,XMMWORD[r8] ; I[0] vpclmulqdq xmm0,xmm14,xmm6,0x00 vpshufb xmm15,xmm15,xmm13 vpclmulqdq xmm1,xmm14,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((160-64))+rdx] + vmovdqu xmm6,XMMWORD[((160-64))+rdx] ; %xmm6^8 vpxor xmm15,xmm15,xmm12 vpclmulqdq xmm2,xmm9,xmm7,0x10 - vpxor xmm15,xmm15,xmm10 + vpxor xmm15,xmm15,xmm10 ; accumulate %xmm10 lea r8,[128+r8] sub r9,0x80 @@ -1039,15 +1039,15 @@ ALIGN 32 $L$short_avx: - vmovdqu xmm14,XMMWORD[((-16))+r9*1+r8] + vmovdqu xmm14,XMMWORD[((-16))+r9*1+r8] ; very last word lea r8,[r9*1+r8] - vmovdqu xmm6,XMMWORD[((0-64))+rdx] + vmovdqu xmm6,XMMWORD[((0-64))+rdx] ; %xmm6^1 vmovdqu xmm7,XMMWORD[((32-64))+rdx] vpshufb xmm15,xmm14,xmm13 - vmovdqa xmm3,xmm0 - vmovdqa xmm4,xmm1 - vmovdqa xmm5,xmm2 + vmovdqa xmm3,xmm0 ; subtle way to zero %xmm3, + vmovdqa xmm4,xmm1 ; %xmm4 and + vmovdqa xmm5,xmm2 ; %xmm5 sub r9,0x10 jz NEAR $L$tail_avx @@ -1058,7 +1058,7 @@ vmovdqu xmm14,XMMWORD[((-32))+r8] vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm1,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((16-64))+rdx] + vmovdqu xmm6,XMMWORD[((16-64))+rdx] ; %xmm6^2 vpshufb xmm15,xmm14,xmm13 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm8,xmm7,0x00 @@ -1073,7 +1073,7 @@ vmovdqu xmm14,XMMWORD[((-48))+r8] vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm1,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((48-64))+rdx] + vmovdqu xmm6,XMMWORD[((48-64))+rdx] ; %xmm6^3 vpshufb xmm15,xmm14,xmm13 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm8,xmm7,0x00 @@ -1088,7 +1088,7 @@ vmovdqu xmm14,XMMWORD[((-64))+r8] vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm1,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((64-64))+rdx] + vmovdqu xmm6,XMMWORD[((64-64))+rdx] ; %xmm6^4 vpshufb xmm15,xmm14,xmm13 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm8,xmm7,0x00 @@ -1103,7 +1103,7 @@ vmovdqu xmm14,XMMWORD[((-80))+r8] vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm1,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((96-64))+rdx] + vmovdqu xmm6,XMMWORD[((96-64))+rdx] ; %xmm6^5 vpshufb xmm15,xmm14,xmm13 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm8,xmm7,0x00 @@ -1118,7 +1118,7 @@ vmovdqu xmm14,XMMWORD[((-96))+r8] vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm1,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((112-64))+rdx] + vmovdqu xmm6,XMMWORD[((112-64))+rdx] ; %xmm6^6 vpshufb xmm15,xmm14,xmm13 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm8,xmm7,0x00 @@ -1133,7 +1133,7 @@ vmovdqu xmm14,XMMWORD[((-112))+r8] vpxor xmm4,xmm4,xmm1 vpclmulqdq xmm1,xmm15,xmm6,0x11 - vmovdqu xmm6,XMMWORD[((144-64))+rdx] + vmovdqu xmm6,XMMWORD[((144-64))+rdx] ; %xmm6^7 vpshufb xmm15,xmm14,xmm13 vpxor xmm5,xmm5,xmm2 vpclmulqdq xmm2,xmm8,xmm7,0x00 @@ -1143,7 +1143,7 @@ ALIGN 32 $L$tail_avx: - vpxor xmm15,xmm15,xmm10 + vpxor xmm15,xmm15,xmm10 ; accumulate %xmm10 $L$tail_no_xor_avx: vpunpckhqdq xmm8,xmm15,xmm15 vpxor xmm3,xmm3,xmm0 @@ -1160,18 +1160,18 @@ vpxor xmm11,xmm4,xmm1 vpxor xmm5,xmm5,xmm2 - vpxor xmm5,xmm5,xmm10 + vpxor xmm5,xmm5,xmm10 ; aggregated Karatsuba post-processing vpxor xmm5,xmm5,xmm11 vpslldq xmm9,xmm5,8 vpsrldq xmm5,xmm5,8 vpxor xmm10,xmm10,xmm9 vpxor xmm11,xmm11,xmm5 - vpclmulqdq xmm9,xmm10,xmm12,0x10 + vpclmulqdq xmm9,xmm10,xmm12,0x10 ; 1st phase vpalignr xmm10,xmm10,xmm10,8 vpxor xmm10,xmm10,xmm9 - vpclmulqdq xmm9,xmm10,xmm12,0x10 + vpclmulqdq xmm9,xmm10,xmm12,0x10 ; 2nd phase vpalignr xmm10,xmm10,xmm10,8 vpxor xmm10,xmm10,xmm11 vpxor xmm10,xmm10,xmm9 @@ -1238,7 +1238,7 @@ section .xdata rdata align=8 ALIGN 4 $L$SEH_info_gcm_init_clmul_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_init_clmul_4-$L$SEH_begin_gcm_init_clmul_1 DB 3 DB 0 @@ -1250,7 +1250,7 @@ DW 0 $L$SEH_info_gcm_ghash_clmul_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_ghash_clmul_13-$L$SEH_begin_gcm_ghash_clmul_1 DB 22 DB 0 @@ -1289,7 +1289,7 @@ DW 21 $L$SEH_info_gcm_init_avx_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_init_avx_4-$L$SEH_begin_gcm_init_avx_1 DB 3 DB 0 @@ -1301,7 +1301,7 @@ DW 0 $L$SEH_info_gcm_ghash_avx_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_gcm_ghash_avx_13-$L$SEH_begin_gcm_ghash_avx_1 DB 22 DB 0
diff --git a/gen/bcm/p256-x86_64-asm-apple.S b/gen/bcm/p256-x86_64-asm-apple.S index 255a34d..eb8000d 100644 --- a/gen/bcm/p256-x86_64-asm-apple.S +++ b/gen/bcm/p256-x86_64-asm-apple.S
@@ -6,7 +6,7 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) .text - +// The polynomial .section __DATA,__const .p2align 6 p256_constants: @@ -22,15 +22,15 @@ L$ONE_mont: .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe - +// Constants for computations modulo ord(p256) L$ord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 L$ordK: .quad 0xccd1c8aaee00bc4f .text - - +// ############################################################################## +// void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]); .globl _ecp_nistz256_neg .private_extern _ecp_nistz256_neg @@ -86,11 +86,11 @@ ret - - - - - +// ############################################################################## +// void ecp_nistz256_ord_mul_mont( +// uint64_t res[4], +// uint64_t a[4], +// uint64_t b[4]); .globl _ecp_nistz256_ord_mul_mont_nohw .private_extern _ecp_nistz256_ord_mul_mont_nohw @@ -119,7 +119,7 @@ leaq L$ord(%rip),%r14 movq L$ordK(%rip),%r15 - +// ################################ * b[0] movq %rax,%rcx mulq 0(%rsi) movq %rax,%r8 @@ -147,16 +147,16 @@ adcq $0,%rdx movq %rdx,%r12 - +// ################################ First reduction step mulq 0(%r14) movq %r8,%rbp - addq %rax,%r13 + addq %rax,%r13 // guaranteed to be zero movq %r8,%rax adcq $0,%rdx movq %rdx,%rcx subq %r8,%r10 - sbbq $0,%r8 + sbbq $0,%r8 // can't borrow mulq 8(%r14) addq %rcx,%r9 @@ -165,19 +165,19 @@ movq %rbp,%rax adcq %rdx,%r10 movq %rbp,%rdx - adcq $0,%r8 + adcq $0,%r8 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r11 movq 8(%rbx),%rax - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r8,%r11 adcq %rbp,%r12 adcq $0,%r13 - +// ################################ * b[1] movq %rax,%rcx mulq 0(%rsi) addq %rax,%r9 @@ -213,15 +213,15 @@ adcq %rdx,%r13 adcq $0,%r8 - +// ################################ Second reduction step mulq 0(%r14) movq %r9,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq %r9,%rax adcq %rdx,%rcx subq %r9,%r11 - sbbq $0,%r9 + sbbq $0,%r9 // can't borrow mulq 8(%r14) addq %rcx,%r10 @@ -230,19 +230,19 @@ movq %rbp,%rax adcq %rdx,%r11 movq %rbp,%rdx - adcq $0,%r9 + adcq $0,%r9 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r12 movq 16(%rbx),%rax - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r9,%r12 adcq %rbp,%r13 adcq $0,%r8 - +// ################################# * b[2] movq %rax,%rcx mulq 0(%rsi) addq %rax,%r10 @@ -278,15 +278,15 @@ adcq %rdx,%r8 adcq $0,%r9 - +// ################################ Third reduction step mulq 0(%r14) movq %r10,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq %r10,%rax adcq %rdx,%rcx subq %r10,%r12 - sbbq $0,%r10 + sbbq $0,%r10 // can't borrow mulq 8(%r14) addq %rcx,%r11 @@ -295,19 +295,19 @@ movq %rbp,%rax adcq %rdx,%r12 movq %rbp,%rdx - adcq $0,%r10 + adcq $0,%r10 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r13 movq 24(%rbx),%rax - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r10,%r13 adcq %rbp,%r8 adcq $0,%r9 - +// ################################ * b[3] movq %rax,%rcx mulq 0(%rsi) addq %rax,%r11 @@ -343,15 +343,15 @@ adcq %rdx,%r9 adcq $0,%r10 - +// ################################ Last reduction step mulq 0(%r14) movq %r11,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq %r11,%rax adcq %rdx,%rcx subq %r11,%r13 - sbbq $0,%r11 + sbbq $0,%r11 // can't borrow mulq 8(%r14) addq %rcx,%r12 @@ -360,18 +360,18 @@ movq %rbp,%rax adcq %rdx,%r13 movq %rbp,%rdx - adcq $0,%r11 + adcq $0,%r11 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r8 - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r11,%r8 adcq %rbp,%r9 adcq $0,%r10 - +// ################################ Subtract ord movq %r12,%rsi subq 0(%r14),%r12 movq %r13,%r11 @@ -411,11 +411,11 @@ - - - - - +// ############################################################################## +// void ecp_nistz256_ord_sqr_mont( +// uint64_t res[4], +// uint64_t a[4], +// uint64_t rep); .globl _ecp_nistz256_ord_sqr_mont_nohw .private_extern _ecp_nistz256_ord_sqr_mont_nohw @@ -443,56 +443,56 @@ movq 8(%rsi),%rax movq 16(%rsi),%r14 movq 24(%rsi),%r15 - leaq L$ord(%rip),%rsi + leaq L$ord(%rip),%rsi // pointer to modulus movq %rdx,%rbx jmp L$oop_ord_sqr .p2align 5 L$oop_ord_sqr: - - movq %rax,%rbp - mulq %r8 +// ################################ a[1:] * a[0] + movq %rax,%rbp // put aside a[1] + mulq %r8 // a[1] * a[0] movq %rax,%r9 - movq %rbp,%xmm1 + movq %rbp,%xmm1 // offload a[1] movq %r14,%rax movq %rdx,%r10 - mulq %r8 + mulq %r8 // a[2] * a[0] addq %rax,%r10 movq %r15,%rax - movq %r14,%xmm2 + movq %r14,%xmm2 // offload a[2] adcq $0,%rdx movq %rdx,%r11 - mulq %r8 + mulq %r8 // a[3] * a[0] addq %rax,%r11 movq %r15,%rax - movq %r15,%xmm3 + movq %r15,%xmm3 // offload a[3] adcq $0,%rdx movq %rdx,%r12 - - mulq %r14 +// ################################ a[3] * a[2] + mulq %r14 // a[3] * a[2] movq %rax,%r13 movq %r14,%rax movq %rdx,%r14 - - mulq %rbp +// ################################ a[2:] * a[1] + mulq %rbp // a[2] * a[1] addq %rax,%r11 movq %r15,%rax adcq $0,%rdx movq %rdx,%r15 - mulq %rbp + mulq %rbp // a[3] * a[1] addq %rax,%r12 adcq $0,%rdx addq %r15,%r12 adcq %rdx,%r13 - adcq $0,%r14 + adcq $0,%r14 // can't overflow - +// ################################ *2 xorq %r15,%r15 movq %r8,%rax addq %r9,%r9 @@ -503,20 +503,20 @@ adcq %r14,%r14 adcq $0,%r15 - - mulq %rax +// ################################ Missing products + mulq %rax // a[0] * a[0] movq %rax,%r8 movq %xmm1,%rax movq %rdx,%rbp - mulq %rax + mulq %rax // a[1] * a[1] addq %rbp,%r9 adcq %rax,%r10 movq %xmm2,%rax adcq $0,%rdx movq %rdx,%rbp - mulq %rax + mulq %rax // a[2] * a[2] addq %rbp,%r11 adcq %rax,%r12 movq %xmm3,%rax @@ -524,23 +524,23 @@ movq %rdx,%rbp movq %r8,%rcx - imulq 32(%rsi),%r8 + imulq 32(%rsi),%r8 // *= .LordK - mulq %rax + mulq %rax // a[3] * a[3] addq %rbp,%r13 adcq %rax,%r14 - movq 0(%rsi),%rax - adcq %rdx,%r15 + movq 0(%rsi),%rax // modulus[0] + adcq %rdx,%r15 // can't overflow - +// ################################ First reduction step mulq %r8 movq %r8,%rbp - addq %rax,%rcx - movq 8(%rsi),%rax + addq %rax,%rcx // guaranteed to be zero + movq 8(%rsi),%rax // modulus[1] adcq %rdx,%rcx subq %r8,%r10 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r8 addq %rcx,%r9 @@ -549,29 +549,29 @@ movq %r8,%rax adcq %rdx,%r10 movq %r8,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow movq %r9,%rcx - imulq 32(%rsi),%r9 + imulq 32(%rsi),%r9 // *= .LordK shlq $32,%rax shrq $32,%rdx subq %rax,%r11 movq 0(%rsi),%rax - sbbq %rdx,%r8 + sbbq %rdx,%r8 // can't borrow addq %rbp,%r11 - adcq $0,%r8 + adcq $0,%r8 // can't overflow - +// ################################ Second reduction step mulq %r9 movq %r9,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq 8(%rsi),%rax adcq %rdx,%rcx subq %r9,%r11 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r9 addq %rcx,%r10 @@ -580,29 +580,29 @@ movq %r9,%rax adcq %rdx,%r11 movq %r9,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow movq %r10,%rcx - imulq 32(%rsi),%r10 + imulq 32(%rsi),%r10 // *= .LordK shlq $32,%rax shrq $32,%rdx subq %rax,%r8 movq 0(%rsi),%rax - sbbq %rdx,%r9 + sbbq %rdx,%r9 // can't borrow addq %rbp,%r8 - adcq $0,%r9 + adcq $0,%r9 // can't overflow - +// ################################ Third reduction step mulq %r10 movq %r10,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq 8(%rsi),%rax adcq %rdx,%rcx subq %r10,%r8 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r10 addq %rcx,%r11 @@ -611,29 +611,29 @@ movq %r10,%rax adcq %rdx,%r8 movq %r10,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow movq %r11,%rcx - imulq 32(%rsi),%r11 + imulq 32(%rsi),%r11 // *= .LordK shlq $32,%rax shrq $32,%rdx subq %rax,%r9 movq 0(%rsi),%rax - sbbq %rdx,%r10 + sbbq %rdx,%r10 // can't borrow addq %rbp,%r9 - adcq $0,%r10 + adcq $0,%r10 // can't overflow - +// ################################ Last reduction step mulq %r11 movq %r11,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq 8(%rsi),%rax adcq %rdx,%rcx subq %r11,%r9 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r11 addq %rcx,%r8 @@ -642,17 +642,17 @@ movq %r11,%rax adcq %rdx,%r9 movq %r11,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r10 - sbbq %rdx,%r11 + sbbq %rdx,%r11 // can't borrow addq %rbp,%r10 - adcq $0,%r11 + adcq $0,%r11 // can't overflow - +// ################################ Add bits [511:256] of the sqr result xorq %rdx,%rdx addq %r12,%r8 adcq %r13,%r9 @@ -662,7 +662,7 @@ movq %r9,%rax adcq $0,%rdx - +// ################################ Compare to modulus subq 0(%rsi),%r8 movq %r10,%r14 sbbq 8(%rsi),%r9 @@ -705,7 +705,7 @@ ret - +// ############################################################################## .globl _ecp_nistz256_ord_mul_mont_adx .private_extern _ecp_nistz256_ord_mul_mont_adx @@ -735,11 +735,11 @@ movq 8(%rsi),%r10 movq 16(%rsi),%r11 movq 24(%rsi),%r12 - leaq -128(%rsi),%rsi + leaq -128(%rsi),%rsi // control u-op density leaq L$ord-128(%rip),%r14 movq L$ordK(%rip),%r15 - +// ################################ Multiply by b[0] mulxq %r9,%r8,%r9 mulxq %r10,%rcx,%r10 mulxq %r11,%rbp,%r11 @@ -751,10 +751,10 @@ adcq %rcx,%r11 adcq $0,%r12 - - xorq %r13,%r13 +// ################################ reduction + xorq %r13,%r13 // %r13=0, cf=0, of=0 mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r8 + adcxq %rcx,%r8 // guaranteed to be zero adoxq %rbp,%r9 mulxq 8+128(%r14),%rcx,%rbp @@ -771,9 +771,9 @@ adoxq %rbp,%r12 adcxq %r8,%r12 adoxq %r8,%r13 - adcq $0,%r13 + adcq $0,%r13 // cf=0, of=0 - +// ################################ Multiply by b[1] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r9 adoxq %rbp,%r10 @@ -794,11 +794,11 @@ adcxq %r8,%r13 adoxq %r8,%r8 - adcq $0,%r8 + adcq $0,%r8 // cf=0, of=0 - +// ################################ reduction mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r9 + adcxq %rcx,%r9 // guaranteed to be zero adoxq %rbp,%r10 mulxq 8+128(%r14),%rcx,%rbp @@ -815,9 +815,9 @@ adoxq %rbp,%r13 adcxq %r9,%r13 adoxq %r9,%r8 - adcq $0,%r8 + adcq $0,%r8 // cf=0, of=0 - +// ################################ Multiply by b[2] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r10 adoxq %rbp,%r11 @@ -838,11 +838,11 @@ adcxq %r9,%r8 adoxq %r9,%r9 - adcq $0,%r9 + adcq $0,%r9 // cf=0, of=0 - +// ################################ reduction mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r10 + adcxq %rcx,%r10 // guaranteed to be zero adoxq %rbp,%r11 mulxq 8+128(%r14),%rcx,%rbp @@ -859,9 +859,9 @@ adoxq %rbp,%r8 adcxq %r10,%r8 adoxq %r10,%r9 - adcq $0,%r9 + adcq $0,%r9 // cf=0, of=0 - +// ################################ Multiply by b[3] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r11 adoxq %rbp,%r12 @@ -882,11 +882,11 @@ adcxq %r10,%r9 adoxq %r10,%r10 - adcq $0,%r10 + adcq $0,%r10 // cf=0, of=0 - +// ################################ reduction mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r11 + adcxq %rcx,%r11 // guaranteed to be zero adoxq %rbp,%r12 mulxq 8+128(%r14),%rcx,%rbp @@ -907,8 +907,8 @@ adoxq %r11,%r10 adcq $0,%r10 - - +// ################################ +// Branch-less conditional subtraction of P movq %r8,%rcx subq 0(%r14),%r12 sbbq 8(%r14),%r13 @@ -979,38 +979,38 @@ .p2align 5 L$oop_ord_sqrx: - mulxq %r14,%r9,%r10 - mulxq %r15,%rcx,%r11 - movq %rdx,%rax - movq %r14,%xmm1 - mulxq %r8,%rbp,%r12 + mulxq %r14,%r9,%r10 // a[0]*a[1] + mulxq %r15,%rcx,%r11 // a[0]*a[2] + movq %rdx,%rax // offload a[0] + movq %r14,%xmm1 // offload a[1] + mulxq %r8,%rbp,%r12 // a[0]*a[3] movq %r14,%rdx addq %rcx,%r10 - movq %r15,%xmm2 + movq %r15,%xmm2 // offload a[2] adcq %rbp,%r11 adcq $0,%r12 - xorq %r13,%r13 - - mulxq %r15,%rcx,%rbp + xorq %r13,%r13 // %r13=0,cf=0,of=0 +// ################################ + mulxq %r15,%rcx,%rbp // a[1]*a[2] adcxq %rcx,%r11 adoxq %rbp,%r12 - mulxq %r8,%rcx,%rbp + mulxq %r8,%rcx,%rbp // a[1]*a[3] movq %r15,%rdx adcxq %rcx,%r12 adoxq %rbp,%r13 adcq $0,%r13 - - mulxq %r8,%rcx,%r14 +// ################################ + mulxq %r8,%rcx,%r14 // a[2]*a[3] movq %rax,%rdx - movq %r8,%xmm3 - xorq %r15,%r15 - adcxq %r9,%r9 + movq %r8,%xmm3 // offload a[3] + xorq %r15,%r15 // %r15=0,cf=0,of=0 + adcxq %r9,%r9 // acc1:6<<1 adoxq %rcx,%r13 adcxq %r10,%r10 - adoxq %r15,%r14 + adoxq %r15,%r14 // of=0 - +// ################################ a[i]*a[i] mulxq %rdx,%r8,%rbp movq %xmm1,%rdx adcxq %r11,%r11 @@ -1032,13 +1032,13 @@ adoxq %rcx,%r14 adoxq %rax,%r15 - +// ################################ reduction movq %r8,%rdx mulxq 32(%rsi),%rdx,%rcx - xorq %rax,%rax + xorq %rax,%rax // cf=0, of=0 mulxq 0(%rsi),%rcx,%rbp - adcxq %rcx,%r8 + adcxq %rcx,%r8 // guaranteed to be zero adoxq %rbp,%r9 mulxq 8(%rsi),%rcx,%rbp adcxq %rcx,%r9 @@ -1048,15 +1048,15 @@ adoxq %rbp,%r11 mulxq 24(%rsi),%rcx,%rbp adcxq %rcx,%r11 - adoxq %rbp,%r8 - adcxq %rax,%r8 + adoxq %rbp,%r8 // of=0 + adcxq %rax,%r8 // cf=0 - +// ################################ movq %r9,%rdx mulxq 32(%rsi),%rdx,%rcx mulxq 0(%rsi),%rcx,%rbp - adoxq %rcx,%r9 + adoxq %rcx,%r9 // guaranteed to be zero adcxq %rbp,%r10 mulxq 8(%rsi),%rcx,%rbp adoxq %rcx,%r10 @@ -1066,15 +1066,15 @@ adcxq %rbp,%r8 mulxq 24(%rsi),%rcx,%rbp adoxq %rcx,%r8 - adcxq %rbp,%r9 - adoxq %rax,%r9 + adcxq %rbp,%r9 // cf=0 + adoxq %rax,%r9 // of=0 - +// ################################ movq %r10,%rdx mulxq 32(%rsi),%rdx,%rcx mulxq 0(%rsi),%rcx,%rbp - adcxq %rcx,%r10 + adcxq %rcx,%r10 // guaranteed to be zero adoxq %rbp,%r11 mulxq 8(%rsi),%rcx,%rbp adcxq %rcx,%r11 @@ -1084,15 +1084,15 @@ adoxq %rbp,%r9 mulxq 24(%rsi),%rcx,%rbp adcxq %rcx,%r9 - adoxq %rbp,%r10 - adcxq %rax,%r10 + adoxq %rbp,%r10 // of=0 + adcxq %rax,%r10 // cf=0 - +// ################################ movq %r11,%rdx mulxq 32(%rsi),%rdx,%rcx mulxq 0(%rsi),%rcx,%rbp - adoxq %rcx,%r11 + adoxq %rcx,%r11 // guaranteed to be zero adcxq %rbp,%r8 mulxq 8(%rsi),%rcx,%rbp adoxq %rcx,%r8 @@ -1105,8 +1105,8 @@ adcxq %rbp,%r11 adoxq %rax,%r11 - - addq %r8,%r12 +// ################################ accumulate upper half + addq %r8,%r12 // add %r12, %r8 adcq %r13,%r9 movq %r12,%rdx adcq %r14,%r10 @@ -1114,7 +1114,7 @@ movq %r9,%r14 adcq $0,%rax - +// ################################ compare to modulus subq 0(%rsi),%r12 movq %r10,%r15 sbbq 8(%rsi),%r9 @@ -1157,11 +1157,11 @@ ret - - - - - +// ############################################################################## +// void ecp_nistz256_mul_mont( +// uint64_t res[4], +// uint64_t a[4], +// uint64_t b[4]); .globl _ecp_nistz256_mul_mont_nohw .private_extern _ecp_nistz256_mul_mont_nohw @@ -1217,8 +1217,8 @@ .alt_entry __ecp_nistz256_mul_montq __ecp_nistz256_mul_montq: - - +// ####################################################################### +// Multiply a by b[0] movq %rax,%rbp mulq %r9 movq L$poly+8(%rip),%r14 @@ -1246,20 +1246,20 @@ xorq %r13,%r13 movq %rdx,%r12 - - - - - - - - +// ####################################################################### +// First reduction step +// Basically now we want to multiply acc[0] by p256, +// and add the result to the acc. +// Due to the special form of p256 we do some optimizations +// +// acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0] +// then we add acc[0] and get acc[0] x 2^96 movq %r8,%rbp shlq $32,%r8 mulq %r15 shrq $32,%rbp - addq %r8,%r9 + addq %r8,%r9 // +=acc[0]<<96 adcq %rbp,%r10 adcq %rax,%r11 movq 8(%rbx),%rax @@ -1267,8 +1267,8 @@ adcq $0,%r13 xorq %r8,%r8 - - +// ####################################################################### +// Multiply by b[1] movq %rax,%rbp mulq 0(%rsi) addq %rax,%r9 @@ -1300,8 +1300,8 @@ adcq %rdx,%r13 adcq $0,%r8 - - +// ####################################################################### +// Second reduction step movq %r9,%rbp shlq $32,%r9 mulq %r15 @@ -1314,8 +1314,8 @@ adcq $0,%r8 xorq %r9,%r9 - - +// ####################################################################### +// Multiply by b[2] movq %rax,%rbp mulq 0(%rsi) addq %rax,%r10 @@ -1347,8 +1347,8 @@ adcq %rdx,%r8 adcq $0,%r9 - - +// ####################################################################### +// Third reduction step movq %r10,%rbp shlq $32,%r10 mulq %r15 @@ -1361,8 +1361,8 @@ adcq $0,%r9 xorq %r10,%r10 - - +// ####################################################################### +// Multiply by b[3] movq %rax,%rbp mulq 0(%rsi) addq %rax,%r11 @@ -1394,8 +1394,8 @@ adcq %rdx,%r9 adcq $0,%r10 - - +// ####################################################################### +// Final reduction step movq %r11,%rbp shlq $32,%r11 mulq %r15 @@ -1408,14 +1408,14 @@ movq %r13,%rbp adcq $0,%r10 - - - subq $-1,%r12 +// ####################################################################### +// Branch-less conditional subtraction of P + subq $-1,%r12 // .Lpoly[0] movq %r8,%rbx - sbbq %r14,%r13 - sbbq $0,%r8 + sbbq %r14,%r13 // .Lpoly[1] + sbbq $0,%r8 // .Lpoly[2] movq %r9,%rdx - sbbq %r15,%r9 + sbbq %r15,%r9 // .Lpoly[3] sbbq $0,%r10 cmovcq %rcx,%r12 @@ -1431,13 +1431,13 @@ +// ############################################################################## +// void ecp_nistz256_sqr_mont( +// uint64_t res[4], +// uint64_t a[4]); - - - - - - +// we optimize the square according to S.Gueron and V.Krasnov, +// "Speeding up Big-Number Squaring" .globl _ecp_nistz256_sqr_mont_nohw .private_extern _ecp_nistz256_sqr_mont_nohw @@ -1491,31 +1491,31 @@ __ecp_nistz256_sqr_montq: movq %rax,%r13 - mulq %r14 + mulq %r14 // a[1]*a[0] movq %rax,%r9 movq %r15,%rax movq %rdx,%r10 - mulq %r13 + mulq %r13 // a[0]*a[2] addq %rax,%r10 movq %r8,%rax adcq $0,%rdx movq %rdx,%r11 - mulq %r13 + mulq %r13 // a[0]*a[3] addq %rax,%r11 movq %r15,%rax adcq $0,%rdx movq %rdx,%r12 - - mulq %r14 +// ################################ + mulq %r14 // a[1]*a[2] addq %rax,%r11 movq %r8,%rax adcq $0,%rdx movq %rdx,%rbp - mulq %r14 + mulq %r14 // a[1]*a[3] addq %rax,%r12 movq %r8,%rax adcq $0,%rdx @@ -1523,15 +1523,15 @@ movq %rdx,%r13 adcq $0,%r13 - - mulq %r15 +// ################################ + mulq %r15 // a[2]*a[3] xorq %r15,%r15 addq %rax,%r13 movq 0(%rsi),%rax movq %rdx,%r14 adcq $0,%r14 - addq %r9,%r9 + addq %r9,%r9 // acc1:6<<1 adcq %r10,%r10 adcq %r11,%r11 adcq %r12,%r12 @@ -1567,21 +1567,21 @@ movq L$poly+8(%rip),%rsi movq L$poly+24(%rip),%rbp - - - +// ######################################### +// Now the reduction +// First iteration movq %r8,%rcx shlq $32,%r8 mulq %rbp shrq $32,%rcx - addq %r8,%r9 + addq %r8,%r9 // +=acc[0]<<96 adcq %rcx,%r10 adcq %rax,%r11 movq %r9,%rax adcq $0,%rdx - - +// ######################################### +// Second iteration movq %r9,%rcx shlq $32,%r9 movq %rdx,%r8 @@ -1593,8 +1593,8 @@ movq %r10,%rax adcq $0,%rdx - - +// ######################################### +// Third iteration movq %r10,%rcx shlq $32,%r10 movq %rdx,%r9 @@ -1606,8 +1606,8 @@ movq %r11,%rax adcq $0,%rdx - - +// ########################################## +// Last iteration movq %r11,%rcx shlq $32,%r11 movq %rdx,%r10 @@ -1619,8 +1619,8 @@ adcq $0,%rdx xorq %r11,%r11 - - +// ########################################### +// Add the rest of the acc addq %r8,%r12 adcq %r9,%r13 movq %r12,%r8 @@ -1629,12 +1629,12 @@ movq %r13,%r9 adcq $0,%r11 - subq $-1,%r12 + subq $-1,%r12 // .Lpoly[0] movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 + sbbq %rsi,%r13 // .Lpoly[1] + sbbq $0,%r14 // .Lpoly[2] movq %r15,%rcx - sbbq %rbp,%r15 + sbbq %rbp,%r15 // .Lpoly[3] sbbq $0,%r11 cmovcq %r8,%r12 @@ -1676,7 +1676,7 @@ movq 8(%rsi),%r10 movq 16(%rsi),%r11 movq 24(%rsi),%r12 - leaq -128(%rsi),%rsi + leaq -128(%rsi),%rsi // control u-op density call __ecp_nistz256_mul_montx @@ -1704,12 +1704,12 @@ .alt_entry __ecp_nistz256_mul_montx __ecp_nistz256_mul_montx: - - +// ####################################################################### +// Multiply by b[0] mulxq %r9,%r8,%r9 mulxq %r10,%rcx,%r10 movq $32,%r14 - xorq %r13,%r13 + xorq %r13,%r13 // cf=0 mulxq %r11,%rbp,%r11 movq L$poly+24(%rip),%r15 adcq %rcx,%r9 @@ -1721,8 +1721,8 @@ shrxq %r14,%r8,%rcx adcq $0,%r12 - - +// ####################################################################### +// First reduction step addq %rbp,%r9 adcq %rcx,%r10 @@ -1731,10 +1731,10 @@ adcq %rcx,%r11 adcq %rbp,%r12 adcq $0,%r13 - xorq %r8,%r8 + xorq %r8,%r8 // %r8=0,cf=0,of=0 - - +// ####################################################################### +// Multiply by b[1] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r9 adoxq %rbp,%r10 @@ -1758,8 +1758,8 @@ adoxq %r8,%r8 adcq $0,%r8 - - +// ####################################################################### +// Second reduction step addq %rcx,%r10 adcq %rbp,%r11 @@ -1768,10 +1768,10 @@ adcq %rcx,%r12 adcq %rbp,%r13 adcq $0,%r8 - xorq %r9,%r9 + xorq %r9,%r9 // %r9=0,cf=0,of=0 - - +// ####################################################################### +// Multiply by b[2] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r10 adoxq %rbp,%r11 @@ -1795,8 +1795,8 @@ adoxq %r9,%r9 adcq $0,%r9 - - +// ####################################################################### +// Third reduction step addq %rcx,%r11 adcq %rbp,%r12 @@ -1805,10 +1805,10 @@ adcq %rcx,%r13 adcq %rbp,%r8 adcq $0,%r9 - xorq %r10,%r10 + xorq %r10,%r10 // %r10=0,cf=0,of=0 - - +// ####################################################################### +// Multiply by b[3] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r11 adoxq %rbp,%r12 @@ -1832,8 +1832,8 @@ adoxq %r10,%r10 adcq $0,%r10 - - +// ####################################################################### +// Fourth reduction step addq %rcx,%r12 adcq %rbp,%r13 @@ -1845,15 +1845,15 @@ adcq %rbp,%r9 adcq $0,%r10 - - +// ####################################################################### +// Branch-less conditional subtraction of P xorl %eax,%eax movq %r8,%rcx - sbbq $-1,%r12 - sbbq %r14,%r13 - sbbq $0,%r8 + sbbq $-1,%r12 // .Lpoly[0] + sbbq %r14,%r13 // .Lpoly[1] + sbbq $0,%r8 // .Lpoly[2] movq %r9,%rbp - sbbq %r15,%r9 + sbbq %r15,%r9 // .Lpoly[3] sbbq $0,%r10 cmovcq %rbx,%r12 @@ -1894,7 +1894,7 @@ movq 8(%rsi),%r14 movq 16(%rsi),%r15 movq 24(%rsi),%r8 - leaq -128(%rsi),%rsi + leaq -128(%rsi),%rsi // control u-op density call __ecp_nistz256_sqr_montx @@ -1922,35 +1922,35 @@ .alt_entry __ecp_nistz256_sqr_montx __ecp_nistz256_sqr_montx: - mulxq %r14,%r9,%r10 - mulxq %r15,%rcx,%r11 + mulxq %r14,%r9,%r10 // a[0]*a[1] + mulxq %r15,%rcx,%r11 // a[0]*a[2] xorl %eax,%eax adcq %rcx,%r10 - mulxq %r8,%rbp,%r12 + mulxq %r8,%rbp,%r12 // a[0]*a[3] movq %r14,%rdx adcq %rbp,%r11 adcq $0,%r12 - xorq %r13,%r13 + xorq %r13,%r13 // %r13=0,cf=0,of=0 - - mulxq %r15,%rcx,%rbp +// ################################ + mulxq %r15,%rcx,%rbp // a[1]*a[2] adcxq %rcx,%r11 adoxq %rbp,%r12 - mulxq %r8,%rcx,%rbp + mulxq %r8,%rcx,%rbp // a[1]*a[3] movq %r15,%rdx adcxq %rcx,%r12 adoxq %rbp,%r13 adcq $0,%r13 - - mulxq %r8,%rcx,%r14 +// ################################ + mulxq %r8,%rcx,%r14 // a[2]*a[3] movq 0+128(%rsi),%rdx - xorq %r15,%r15 - adcxq %r9,%r9 + xorq %r15,%r15 // %r15=0,cf=0,of=0 + adcxq %r9,%r9 // acc1:6<<1 adoxq %rcx,%r13 adcxq %r10,%r10 - adoxq %r15,%r14 + adoxq %r15,%r14 // of=0 mulxq %rdx,%r8,%rbp movq 8+128(%rsi),%rdx @@ -1979,7 +1979,7 @@ shrxq %rsi,%r8,%rax movq %rdx,%rbp - +// reduction step 1 addq %rcx,%r9 adcq %rax,%r10 @@ -1989,7 +1989,7 @@ adcq $0,%r8 shrxq %rsi,%r9,%rax - +// reduction step 2 addq %rcx,%r10 adcq %rax,%r11 @@ -1999,7 +1999,7 @@ adcq $0,%r9 shrxq %rsi,%r10,%rax - +// reduction step 3 addq %rcx,%r11 adcq %rax,%r8 @@ -2009,7 +2009,7 @@ adcq $0,%r10 shrxq %rsi,%r11,%rax - +// reduction step 4 addq %rcx,%r8 adcq %rax,%r9 @@ -2018,7 +2018,7 @@ adcq $0,%r11 xorq %rdx,%rdx - addq %r8,%r12 + addq %r8,%r12 // accumulate upper half movq L$poly+8(%rip),%rsi adcq %r9,%r13 movq %r12,%r8 @@ -2027,12 +2027,12 @@ movq %r13,%r9 adcq $0,%rdx - subq $-1,%r12 + subq $-1,%r12 // .Lpoly[0] movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 + sbbq %rsi,%r13 // .Lpoly[1] + sbbq $0,%r14 // .Lpoly[2] movq %r15,%r11 - sbbq %rbp,%r15 + sbbq %rbp,%r15 // .Lpoly[3] sbbq $0,%rdx cmovcq %r8,%r12 @@ -2047,8 +2047,8 @@ ret - - +// ############################################################################## +// void ecp_nistz256_select_w5_nohw(uint64_t *val, uint64_t *in_t, int index); .globl _ecp_nistz256_select_w5_nohw .private_extern _ecp_nistz256_select_w5_nohw @@ -2112,8 +2112,8 @@ L$SEH_end_ecp_nistz256_select_w5_nohw: - - +// ############################################################################## +// void ecp_nistz256_select_w7_nohw(uint64_t *val, uint64_t *in_t, int index); .globl _ecp_nistz256_select_w7_nohw .private_extern _ecp_nistz256_select_w7_nohw @@ -2165,8 +2165,8 @@ L$SEH_end_ecp_nistz256_select_w7_nohw: - - +// ############################################################################## +// void ecp_nistz256_select_w5_avx2(uint64_t *val, uint64_t *in_t, int index); .globl _ecp_nistz256_select_w5_avx2 .private_extern _ecp_nistz256_select_w5_avx2 @@ -2232,8 +2232,8 @@ L$SEH_end_ecp_nistz256_select_w5_avx2: - - +// ############################################################################## +// void ecp_nistz256_select_w7_avx2(uint64_t *val, uint64_t *in_t, int index); .globl _ecp_nistz256_select_w7_avx2 .private_extern _ecp_nistz256_select_w7_avx2 @@ -2254,7 +2254,7 @@ vmovd %edx,%xmm1 vpermd %ymm1,%ymm2,%ymm1 - +// Skip index = 0, because it is implicitly the point at infinity movq $21,%rax L$select_loop_avx2_w7: @@ -2418,7 +2418,7 @@ __ecp_nistz256_mul_by_2q: xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 adcq %r13,%r13 movq %r12,%rax adcq %r8,%r8 @@ -2471,10 +2471,10 @@ L$point_doubleq_body: L$point_double_shortcutq: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi.x + movq %rsi,%rbx // backup copy movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 + movq 32+0(%rsi),%r12 // load in_y in "5-4-0-1" order movq 32+8(%rsi),%r13 movq 32+16(%rsi),%r8 movq 32+24(%rsi),%r9 @@ -2489,7 +2489,7 @@ movq %r11,%xmm2 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_by_2q // p256_mul_by_2(S, in_y); movq 64+0(%rsi),%rax movq 64+8(%rsi),%r14 @@ -2497,7 +2497,7 @@ movq 64+24(%rsi),%r8 leaq 64-0(%rsi),%rsi leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Zsqr, in_z); movq 0+0(%rsp),%rax movq 8+0(%rsp),%r14 @@ -2505,9 +2505,9 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(S, S); - movq 32(%rbx),%rax + movq 32(%rbx),%rax // %rbx is still valid movq 64+0(%rbx),%r9 movq 64+8(%rbx),%r10 movq 64+16(%rbx),%r11 @@ -2515,24 +2515,24 @@ leaq 64-0(%rbx),%rsi leaq 32(%rbx),%rbx movq %xmm2,%rdi - call __ecp_nistz256_mul_montq - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2q // p256_mul_by_2(res_z, res_z); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq + call __ecp_nistz256_add_toq // p256_add(M, in_x, Zsqr); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(Zsqr, in_x, Zsqr); movq 0+0(%rsp),%rax movq 8+0(%rsp),%r14 @@ -2540,7 +2540,7 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 movq %xmm1,%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(res_y, S); xorq %r9,%r9 movq %r12,%rax addq $-1,%r12 @@ -2551,7 +2551,7 @@ movq %r15,%r8 adcq %rbp,%r15 adcq $0,%r9 - xorq %rsi,%rsi + xorq %rsi,%rsi // borrow %rsi testq $1,%rax cmovzq %rax,%r12 @@ -2560,7 +2560,7 @@ cmovzq %r8,%r15 cmovzq %rsi,%r9 - movq %r13,%rax + movq %r13,%rax // a0:a3>>1 shrq $1,%r12 shlq $63,%rax movq %r14,%r10 @@ -2587,14 +2587,14 @@ movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(M, M, Zsqr); leaq 128(%rsp),%rdi call __ecp_nistz256_mul_by_2q leaq 32(%rsp),%rbx leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq + call __ecp_nistz256_add_toq // p256_mul_by_3(M, M); movq 96(%rsp),%rax leaq 96(%rsp),%rbx @@ -2604,10 +2604,10 @@ movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S, S, in_x); leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_by_2q // p256_mul_by_2(tmp0, S); movq 0+32(%rsp),%rax movq 8+32(%rsp),%r14 @@ -2615,27 +2615,27 @@ movq 16+32(%rsp),%r15 movq 24+32(%rsp),%r8 movq %xmm0,%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(res_x, M); leaq 128(%rsp),%rbx - movq %r14,%r8 + movq %r14,%r8 // harmonize sqr output and sub input movq %r15,%r9 movq %rsi,%r14 movq %rbp,%r15 - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_x, res_x, tmp0); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 + movq 0+24(%rsp),%r10 // "4-5-0-1" order leaq 0(%rsp),%rdi - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(S, S, res_x); movq 32(%rsp),%rax leaq 32(%rsp),%rbx - movq %r12,%r14 + movq %r12,%r14 // harmonize sub output and mul input xorl %ecx,%ecx - movq %r12,0+0(%rsp) + movq %r12,0+0(%rsp) // have to save:-( movq %r13,%r10 movq %r13,0+8(%rsp) cmovzq %r8,%r11 @@ -2645,11 +2645,11 @@ movq %r9,0+24(%rsp) movq %r14,%r9 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S, S, M); movq %xmm1,%rbx movq %xmm1,%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_y, S, res_y); leaq 160+56(%rsp),%rsi @@ -2695,14 +2695,14 @@ L$point_addq_body: - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi + movq %rsi,%rbx // reassign + movq %rdx,%rsi // reassign movdqa %xmm0,384(%rsp) movdqa %xmm1,384+16(%rsp) movdqa %xmm2,416(%rsp) @@ -2711,54 +2711,54 @@ movdqa %xmm5,448+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 por %xmm3,%xmm5 movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rax + movq 64+0(%rsi),%rax // load original in2_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 movdqa %xmm0,480(%rsp) pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 + movdqu 64(%rsi),%xmm0 // in2_z again movdqu 80(%rsi),%xmm1 movdqa %xmm2,512(%rsp) movdqa %xmm3,512+16(%rsp) por %xmm4,%xmm5 pxor %xmm4,%xmm4 por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi - leaq 64-0(%rsi),%rsi - movq %rax,544+0(%rsp) + leaq 64-0(%rsi),%rsi // %rsi is still valid + movq %rax,544+0(%rsp) // make in2_z copy movq %r14,544+8(%rsp) movq %r15,544+16(%rsp) movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 96(%rsp),%rdi // Z2^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Z2sqr, in2_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm1,%xmm4 por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 por %xmm3,%xmm4 pxor %xmm3,%xmm3 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rax + pshufd $0,%xmm4,%xmm4 // in2infty + movq 64+0(%rbx),%rax // load original in1_z movq 64+8(%rbx),%r14 movq 64+16(%rbx),%r15 movq 64+24(%rbx),%r8 movq %rbx,%xmm1 leaq 64-0(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Z1sqr, in1_z); movq 544(%rsp),%rax leaq 544(%rsp),%rbx @@ -2767,8 +2767,8 @@ leaq 0+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 224(%rsp),%rdi // S1 = Z2^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S1, Z2sqr, in2_z); movq 448(%rsp),%rax leaq 448(%rsp),%rbx @@ -2777,8 +2777,8 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 256(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, Z1sqr, in1_z); movq 416(%rsp),%rax leaq 416(%rsp),%rbx @@ -2787,8 +2787,8 @@ leaq 0+224(%rsp),%rsi movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 224(%rsp),%rdi // S1 = Y1*Z2^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S1, S1, in1_y); movq 512(%rsp),%rax leaq 512(%rsp),%rbx @@ -2797,18 +2797,18 @@ leaq 0+256(%rsp),%rsi movq 16+256(%rsp),%r11 movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 256(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, S2, in2_y); leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 64(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromq // p256_sub(R, S2, S1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero movdqa %xmm4,%xmm2 orq %r8,%r12 orq %r9,%r12 - por %xmm5,%xmm2 + por %xmm5,%xmm2 // in1infty || in2infty movq %r12,%xmm3 movq 384(%rsp),%rax @@ -2818,8 +2818,8 @@ leaq 0+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 160(%rsp),%rdi // U1 = X1*Z2^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U1, in1_x, Z2sqr); movq 480(%rsp),%rax leaq 480(%rsp),%rbx @@ -2828,34 +2828,34 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 192(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, in2_x, Z1sqr); leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 0(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromq // p256_sub(H, U2, U1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero orq %r8,%r12 - orq %r9,%r12 + orq %r9,%r12 // movq %xmm2,%r8 movq %xmm3,%r9 orq %r8,%r12 -.byte 0x3e - jnz L$add_proceedq +.byte 0x3e // predict taken + jnz L$add_proceedq // - - +// We now know A = B or A = -B and neither is infinity. Compare the +// y-coordinates via S1 and S2. testq %r9,%r9 - jz L$add_doubleq + jz L$add_doubleq // is_equal(S1, S2) - - - - - - movq %xmm0,%rdi +// A = -B, so the result is infinity. +// +// TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in +// which case we should eliminate this special-case and simplify the +// timing analysis. + movq %xmm0,%rdi // restore %rdi pxor %xmm0,%xmm0 movdqu %xmm0,0(%rdi) movdqu %xmm0,16(%rdi) @@ -2867,9 +2867,9 @@ .p2align 5 L$add_doubleq: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp + movq %xmm1,%rsi // restore %rsi + movq %xmm0,%rdi // restore %rdi + addq $416,%rsp // difference in frame sizes jmp L$point_double_shortcutq @@ -2881,8 +2881,8 @@ leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 96(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Rsqr, R); movq 448(%rsp),%rax leaq 448(%rsp),%rbx @@ -2891,16 +2891,16 @@ leaq 0+0(%rsp),%rsi movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, H, in1_z); movq 0+0(%rsp),%rax movq 8+0(%rsp),%r14 leaq 0+0(%rsp),%rsi movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 32(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Hsqr, H); movq 544(%rsp),%rax leaq 544(%rsp),%rbx @@ -2909,8 +2909,8 @@ leaq 0+352(%rsp),%rsi movq 16+352(%rsp),%r11 movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, res_z, in2_z); movq 0(%rsp),%rax leaq 0(%rsp),%rbx @@ -2919,8 +2919,8 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 128(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(Hcub, Hsqr, H); movq 160(%rsp),%rax leaq 160(%rsp),%rbx @@ -2929,14 +2929,14 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - + leaq 192(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, U1, Hsqr); +// lea 192(%rsp), %rsi +// lea 32(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 96(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -2962,11 +2962,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(res_x, Rsqr, Hsqr); leaq 128(%rsp),%rbx leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_x, res_x, Hcub); movq 192+0(%rsp),%rax movq 192+8(%rsp),%rbp @@ -2974,10 +2974,10 @@ movq 192+24(%rsp),%r10 leaq 320(%rsp),%rdi - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(res_y, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 128(%rsp),%rax @@ -2988,7 +2988,7 @@ movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, S1, Hcub); movq 320(%rsp),%rax leaq 320(%rsp),%rbx @@ -2998,15 +2998,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(res_y, R, res_y); leaq 256(%rsp),%rbx leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_y, res_y, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, in2_z, in1infty); movdqa %xmm5,%xmm1 pandn 352(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3017,7 +3017,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3030,7 +3030,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3041,7 +3041,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3054,7 +3054,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 320(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3065,7 +3065,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3123,14 +3123,14 @@ L$add_affineq_body: - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi + movq %rdx,%rbx // reassign movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rax + movq 64+0(%rsi),%rax // load original in1_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 @@ -3142,7 +3142,7 @@ movdqa %xmm5,384+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rbx),%xmm0 + movdqu 0(%rbx),%xmm0 // copy *(P256_POINT_AFFINE *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rbx),%xmm1 movdqu 32(%rbx),%xmm2 @@ -3152,7 +3152,7 @@ pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,416+16(%rsp) por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi movdqa %xmm2,448(%rsp) movdqa %xmm3,448+16(%rsp) por %xmm2,%xmm3 @@ -3160,33 +3160,33 @@ pxor %xmm4,%xmm4 por %xmm1,%xmm3 - leaq 64-0(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 64-0(%rsi),%rsi // %rsi is still valid + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Z1sqr, in1_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rax - - movq %r12,%r9 + movq 0(%rbx),%rax // %rbx is still valid +// lea 0x00(%rbx), %rbx + movq %r12,%r9 // harmonize sqr output and mul input por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 movq %r13,%r10 por %xmm3,%xmm4 pxor %xmm3,%xmm3 movq %r14,%r11 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 + pshufd $0,%xmm4,%xmm4 // in2infty leaq 32-0(%rsp),%rsi movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 0(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, Z1sqr, in2_x); leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 64(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromq // p256_sub(H, U2, in1_x); movq 384(%rsp),%rax leaq 384(%rsp),%rbx @@ -3195,8 +3195,8 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 32(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, Z1sqr, in1_z); movq 384(%rsp),%rax leaq 384(%rsp),%rbx @@ -3205,8 +3205,8 @@ leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 288(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, H, in1_z); movq 448(%rsp),%rax leaq 448(%rsp),%rbx @@ -3215,28 +3215,28 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 32(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, S2, in2_y); leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 96(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromq // p256_sub(R, S2, in1_y); movq 0+64(%rsp),%rax movq 8+64(%rsp),%r14 leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 128(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Hsqr, H); movq 0+96(%rsp),%rax movq 8+96(%rsp),%r14 leaq 0+96(%rsp),%rsi movq 16+96(%rsp),%r15 movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 192(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Rsqr, R); movq 128(%rsp),%rax leaq 128(%rsp),%rbx @@ -3245,8 +3245,8 @@ leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 160(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(Hcub, Hsqr, H); movq 320(%rsp),%rax leaq 320(%rsp),%rbx @@ -3255,14 +3255,14 @@ leaq 0+128(%rsp),%rsi movq 16+128(%rsp),%r11 movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - + leaq 0(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, in1_x, Hsqr); +// lea 0(%rsp), %rsi +// lea 128(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 192(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -3288,11 +3288,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(res_x, Rsqr, Hsqr); leaq 160(%rsp),%rbx leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_x, res_x, Hcub); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp @@ -3300,10 +3300,10 @@ movq 0+24(%rsp),%r10 leaq 64(%rsp),%rdi - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(H, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 352(%rsp),%rax @@ -3314,7 +3314,7 @@ movq 16+160(%rsp),%r11 movq 24+160(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, Hcub, in1_y); movq 96(%rsp),%rax leaq 96(%rsp),%rbx @@ -3324,15 +3324,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(H, H, R); leaq 32(%rsp),%rbx leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_y, H, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, ONE, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3343,7 +3343,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3356,7 +3356,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 224(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3367,7 +3367,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3380,7 +3380,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 256(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3391,7 +3391,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3534,7 +3534,7 @@ __ecp_nistz256_mul_by_2x: xorq %r11,%r11 - adcq %r12,%r12 + adcq %r12,%r12 // a0:a3+a0:a3 adcq %r13,%r13 movq %r12,%rax adcq %r8,%r8 @@ -3588,10 +3588,10 @@ L$point_doublex_body: L$point_double_shortcutx: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi.x + movq %rsi,%rbx // backup copy movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 + movq 32+0(%rsi),%r12 // load in_y in "5-4-0-1" order movq 32+8(%rsi),%r13 movq 32+16(%rsi),%r8 movq 32+24(%rsi),%r9 @@ -3606,7 +3606,7 @@ movq %r11,%xmm2 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_by_2x // p256_mul_by_2(S, in_y); movq 64+0(%rsi),%rdx movq 64+8(%rsi),%r14 @@ -3614,7 +3614,7 @@ movq 64+24(%rsi),%r8 leaq 64-128(%rsi),%rsi leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Zsqr, in_z); movq 0+0(%rsp),%rdx movq 8+0(%rsp),%r14 @@ -3622,9 +3622,9 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(S, S); - movq 32(%rbx),%rdx + movq 32(%rbx),%rdx // %rbx is still valid movq 64+0(%rbx),%r9 movq 64+8(%rbx),%r10 movq 64+16(%rbx),%r11 @@ -3632,24 +3632,24 @@ leaq 64-128(%rbx),%rsi leaq 32(%rbx),%rbx movq %xmm2,%rdi - call __ecp_nistz256_mul_montx - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2x // p256_mul_by_2(res_z, res_z); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox + call __ecp_nistz256_add_tox // p256_add(M, in_x, Zsqr); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(Zsqr, in_x, Zsqr); movq 0+0(%rsp),%rdx movq 8+0(%rsp),%r14 @@ -3657,7 +3657,7 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 movq %xmm1,%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(res_y, S); xorq %r9,%r9 movq %r12,%rax addq $-1,%r12 @@ -3668,7 +3668,7 @@ movq %r15,%r8 adcq %rbp,%r15 adcq $0,%r9 - xorq %rsi,%rsi + xorq %rsi,%rsi // borrow %rsi testq $1,%rax cmovzq %rax,%r12 @@ -3677,7 +3677,7 @@ cmovzq %r8,%r15 cmovzq %rsi,%r9 - movq %r13,%rax + movq %r13,%rax // a0:a3>>1 shrq $1,%r12 shlq $63,%rax movq %r14,%r10 @@ -3704,14 +3704,14 @@ movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(M, M, Zsqr); leaq 128(%rsp),%rdi call __ecp_nistz256_mul_by_2x leaq 32(%rsp),%rbx leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox + call __ecp_nistz256_add_tox // p256_mul_by_3(M, M); movq 96(%rsp),%rdx leaq 96(%rsp),%rbx @@ -3721,10 +3721,10 @@ movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S, S, in_x); leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_by_2x // p256_mul_by_2(tmp0, S); movq 0+32(%rsp),%rdx movq 8+32(%rsp),%r14 @@ -3732,27 +3732,27 @@ movq 16+32(%rsp),%r15 movq 24+32(%rsp),%r8 movq %xmm0,%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(res_x, M); leaq 128(%rsp),%rbx - movq %r14,%r8 + movq %r14,%r8 // harmonize sqr output and sub input movq %r15,%r9 movq %rsi,%r14 movq %rbp,%r15 - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_x, res_x, tmp0); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 + movq 0+24(%rsp),%r10 // "4-5-0-1" order leaq 0(%rsp),%rdi - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(S, S, res_x); movq 32(%rsp),%rdx leaq 32(%rsp),%rbx - movq %r12,%r14 + movq %r12,%r14 // harmonize sub output and mul input xorl %ecx,%ecx - movq %r12,0+0(%rsp) + movq %r12,0+0(%rsp) // have to save:-( movq %r13,%r10 movq %r13,0+8(%rsp) cmovzq %r8,%r11 @@ -3762,11 +3762,11 @@ movq %r9,0+24(%rsp) movq %r14,%r9 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S, S, M); movq %xmm1,%rbx movq %xmm1,%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_y, S, res_y); leaq 160+56(%rsp),%rsi @@ -3812,14 +3812,14 @@ L$point_addx_body: - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi + movq %rsi,%rbx // reassign + movq %rdx,%rsi // reassign movdqa %xmm0,384(%rsp) movdqa %xmm1,384+16(%rsp) movdqa %xmm2,416(%rsp) @@ -3828,54 +3828,54 @@ movdqa %xmm5,448+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 por %xmm3,%xmm5 movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rdx + movq 64+0(%rsi),%rdx // load original in2_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 movdqa %xmm0,480(%rsp) pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 + movdqu 64(%rsi),%xmm0 // in2_z again movdqu 80(%rsi),%xmm1 movdqa %xmm2,512(%rsp) movdqa %xmm3,512+16(%rsp) por %xmm4,%xmm5 pxor %xmm4,%xmm4 por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi - leaq 64-128(%rsi),%rsi - movq %rdx,544+0(%rsp) + leaq 64-128(%rsi),%rsi // %rsi is still valid + movq %rdx,544+0(%rsp) // make in2_z copy movq %r14,544+8(%rsp) movq %r15,544+16(%rsp) movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 96(%rsp),%rdi // Z2^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Z2sqr, in2_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm1,%xmm4 por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 por %xmm3,%xmm4 pxor %xmm3,%xmm3 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rdx + pshufd $0,%xmm4,%xmm4 // in2infty + movq 64+0(%rbx),%rdx // load original in1_z movq 64+8(%rbx),%r14 movq 64+16(%rbx),%r15 movq 64+24(%rbx),%r8 movq %rbx,%xmm1 leaq 64-128(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Z1sqr, in1_z); movq 544(%rsp),%rdx leaq 544(%rsp),%rbx @@ -3884,8 +3884,8 @@ leaq -128+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 224(%rsp),%rdi // S1 = Z2^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S1, Z2sqr, in2_z); movq 448(%rsp),%rdx leaq 448(%rsp),%rbx @@ -3894,8 +3894,8 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 256(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, Z1sqr, in1_z); movq 416(%rsp),%rdx leaq 416(%rsp),%rbx @@ -3904,8 +3904,8 @@ leaq -128+224(%rsp),%rsi movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 224(%rsp),%rdi // S1 = Y1*Z2^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S1, S1, in1_y); movq 512(%rsp),%rdx leaq 512(%rsp),%rbx @@ -3914,18 +3914,18 @@ leaq -128+256(%rsp),%rsi movq 16+256(%rsp),%r11 movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 256(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, S2, in2_y); leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 64(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromx // p256_sub(R, S2, S1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero movdqa %xmm4,%xmm2 orq %r8,%r12 orq %r9,%r12 - por %xmm5,%xmm2 + por %xmm5,%xmm2 // in1infty || in2infty movq %r12,%xmm3 movq 384(%rsp),%rdx @@ -3935,8 +3935,8 @@ leaq -128+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 160(%rsp),%rdi // U1 = X1*Z2^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U1, in1_x, Z2sqr); movq 480(%rsp),%rdx leaq 480(%rsp),%rbx @@ -3945,34 +3945,34 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 192(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, in2_x, Z1sqr); leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 0(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromx // p256_sub(H, U2, U1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero orq %r8,%r12 - orq %r9,%r12 + orq %r9,%r12 // movq %xmm2,%r8 movq %xmm3,%r9 orq %r8,%r12 -.byte 0x3e - jnz L$add_proceedx +.byte 0x3e // predict taken + jnz L$add_proceedx // - - +// We now know A = B or A = -B and neither is infinity. Compare the +// y-coordinates via S1 and S2. testq %r9,%r9 - jz L$add_doublex + jz L$add_doublex // is_equal(S1, S2) - - - - - - movq %xmm0,%rdi +// A = -B, so the result is infinity. +// +// TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in +// which case we should eliminate this special-case and simplify the +// timing analysis. + movq %xmm0,%rdi // restore %rdi pxor %xmm0,%xmm0 movdqu %xmm0,0(%rdi) movdqu %xmm0,16(%rdi) @@ -3984,9 +3984,9 @@ .p2align 5 L$add_doublex: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp + movq %xmm1,%rsi // restore %rsi + movq %xmm0,%rdi // restore %rdi + addq $416,%rsp // difference in frame sizes jmp L$point_double_shortcutx @@ -3998,8 +3998,8 @@ leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 96(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Rsqr, R); movq 448(%rsp),%rdx leaq 448(%rsp),%rbx @@ -4008,16 +4008,16 @@ leaq -128+0(%rsp),%rsi movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, H, in1_z); movq 0+0(%rsp),%rdx movq 8+0(%rsp),%r14 leaq -128+0(%rsp),%rsi movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 32(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Hsqr, H); movq 544(%rsp),%rdx leaq 544(%rsp),%rbx @@ -4026,8 +4026,8 @@ leaq -128+352(%rsp),%rsi movq 16+352(%rsp),%r11 movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, res_z, in2_z); movq 0(%rsp),%rdx leaq 0(%rsp),%rbx @@ -4036,8 +4036,8 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 128(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(Hcub, Hsqr, H); movq 160(%rsp),%rdx leaq 160(%rsp),%rbx @@ -4046,14 +4046,14 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - + leaq 192(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, U1, Hsqr); +// lea 192(%rsp), %rsi +// lea 32(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 96(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -4079,11 +4079,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(res_x, Rsqr, Hsqr); leaq 128(%rsp),%rbx leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_x, res_x, Hcub); movq 192+0(%rsp),%rax movq 192+8(%rsp),%rbp @@ -4091,10 +4091,10 @@ movq 192+24(%rsp),%r10 leaq 320(%rsp),%rdi - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(res_y, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 128(%rsp),%rdx @@ -4105,7 +4105,7 @@ movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, S1, Hcub); movq 320(%rsp),%rdx leaq 320(%rsp),%rbx @@ -4115,15 +4115,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(res_y, R, res_y); leaq 256(%rsp),%rbx leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_y, res_y, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, in2_z, in1infty); movdqa %xmm5,%xmm1 pandn 352(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4134,7 +4134,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4147,7 +4147,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4158,7 +4158,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4171,7 +4171,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 320(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4182,7 +4182,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4240,14 +4240,14 @@ L$add_affinex_body: - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi + movq %rdx,%rbx // reassign movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rdx + movq 64+0(%rsi),%rdx // load original in1_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 @@ -4259,7 +4259,7 @@ movdqa %xmm5,384+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rbx),%xmm0 + movdqu 0(%rbx),%xmm0 // copy *(P256_POINT_AFFINE *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rbx),%xmm1 movdqu 32(%rbx),%xmm2 @@ -4269,7 +4269,7 @@ pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,416+16(%rsp) por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi movdqa %xmm2,448(%rsp) movdqa %xmm3,448+16(%rsp) por %xmm2,%xmm3 @@ -4277,33 +4277,33 @@ pxor %xmm4,%xmm4 por %xmm1,%xmm3 - leaq 64-128(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 64-128(%rsi),%rsi // %rsi is still valid + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Z1sqr, in1_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rdx - - movq %r12,%r9 + movq 0(%rbx),%rdx // %rbx is still valid +// lea 0x00(%rbx), %rbx + movq %r12,%r9 // harmonize sqr output and mul input por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 movq %r13,%r10 por %xmm3,%xmm4 pxor %xmm3,%xmm3 movq %r14,%r11 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 + pshufd $0,%xmm4,%xmm4 // in2infty leaq 32-128(%rsp),%rsi movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 0(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, Z1sqr, in2_x); leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 64(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromx // p256_sub(H, U2, in1_x); movq 384(%rsp),%rdx leaq 384(%rsp),%rbx @@ -4312,8 +4312,8 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 32(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, Z1sqr, in1_z); movq 384(%rsp),%rdx leaq 384(%rsp),%rbx @@ -4322,8 +4322,8 @@ leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 288(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, H, in1_z); movq 448(%rsp),%rdx leaq 448(%rsp),%rbx @@ -4332,28 +4332,28 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 32(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, S2, in2_y); leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 96(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromx // p256_sub(R, S2, in1_y); movq 0+64(%rsp),%rdx movq 8+64(%rsp),%r14 leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 128(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Hsqr, H); movq 0+96(%rsp),%rdx movq 8+96(%rsp),%r14 leaq -128+96(%rsp),%rsi movq 16+96(%rsp),%r15 movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 192(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Rsqr, R); movq 128(%rsp),%rdx leaq 128(%rsp),%rbx @@ -4362,8 +4362,8 @@ leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 160(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(Hcub, Hsqr, H); movq 320(%rsp),%rdx leaq 320(%rsp),%rbx @@ -4372,14 +4372,14 @@ leaq -128+128(%rsp),%rsi movq 16+128(%rsp),%r11 movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - + leaq 0(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, in1_x, Hsqr); +// lea 0(%rsp), %rsi +// lea 128(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 192(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -4405,11 +4405,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(res_x, Rsqr, Hsqr); leaq 160(%rsp),%rbx leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_x, res_x, Hcub); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp @@ -4417,10 +4417,10 @@ movq 0+24(%rsp),%r10 leaq 64(%rsp),%rdi - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(H, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 352(%rsp),%rdx @@ -4431,7 +4431,7 @@ movq 16+160(%rsp),%r11 movq 24+160(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, Hcub, in1_y); movq 96(%rsp),%rdx leaq 96(%rsp),%rbx @@ -4441,15 +4441,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(H, H, R); leaq 32(%rsp),%rbx leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_y, H, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, ONE, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4460,7 +4460,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4473,7 +4473,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 224(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4484,7 +4484,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4497,7 +4497,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 256(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4508,7 +4508,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2
diff --git a/gen/bcm/p256-x86_64-asm-linux.S b/gen/bcm/p256-x86_64-asm-linux.S index 059e4cf..c155c96 100644 --- a/gen/bcm/p256-x86_64-asm-linux.S +++ b/gen/bcm/p256-x86_64-asm-linux.S
@@ -6,7 +6,7 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) .text - +// The polynomial .section .rodata .align 64 p256_constants: @@ -22,15 +22,15 @@ .LONE_mont: .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe - +// Constants for computations modulo ord(p256) .Lord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 .LordK: .quad 0xccd1c8aaee00bc4f .text - - +// ############################################################################## +// void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]); .globl ecp_nistz256_neg .hidden ecp_nistz256_neg .type ecp_nistz256_neg,@function @@ -88,11 +88,11 @@ ret .cfi_endproc .size ecp_nistz256_neg,.-ecp_nistz256_neg - - - - - +// ############################################################################## +// void ecp_nistz256_ord_mul_mont( +// uint64_t res[4], +// uint64_t a[4], +// uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont_nohw .hidden ecp_nistz256_ord_mul_mont_nohw @@ -126,7 +126,7 @@ leaq .Lord(%rip),%r14 movq .LordK(%rip),%r15 - +// ################################ * b[0] movq %rax,%rcx mulq 0(%rsi) movq %rax,%r8 @@ -154,16 +154,16 @@ adcq $0,%rdx movq %rdx,%r12 - +// ################################ First reduction step mulq 0(%r14) movq %r8,%rbp - addq %rax,%r13 + addq %rax,%r13 // guaranteed to be zero movq %r8,%rax adcq $0,%rdx movq %rdx,%rcx subq %r8,%r10 - sbbq $0,%r8 + sbbq $0,%r8 // can't borrow mulq 8(%r14) addq %rcx,%r9 @@ -172,19 +172,19 @@ movq %rbp,%rax adcq %rdx,%r10 movq %rbp,%rdx - adcq $0,%r8 + adcq $0,%r8 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r11 movq 8(%rbx),%rax - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r8,%r11 adcq %rbp,%r12 adcq $0,%r13 - +// ################################ * b[1] movq %rax,%rcx mulq 0(%rsi) addq %rax,%r9 @@ -220,15 +220,15 @@ adcq %rdx,%r13 adcq $0,%r8 - +// ################################ Second reduction step mulq 0(%r14) movq %r9,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq %r9,%rax adcq %rdx,%rcx subq %r9,%r11 - sbbq $0,%r9 + sbbq $0,%r9 // can't borrow mulq 8(%r14) addq %rcx,%r10 @@ -237,19 +237,19 @@ movq %rbp,%rax adcq %rdx,%r11 movq %rbp,%rdx - adcq $0,%r9 + adcq $0,%r9 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r12 movq 16(%rbx),%rax - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r9,%r12 adcq %rbp,%r13 adcq $0,%r8 - +// ################################# * b[2] movq %rax,%rcx mulq 0(%rsi) addq %rax,%r10 @@ -285,15 +285,15 @@ adcq %rdx,%r8 adcq $0,%r9 - +// ################################ Third reduction step mulq 0(%r14) movq %r10,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq %r10,%rax adcq %rdx,%rcx subq %r10,%r12 - sbbq $0,%r10 + sbbq $0,%r10 // can't borrow mulq 8(%r14) addq %rcx,%r11 @@ -302,19 +302,19 @@ movq %rbp,%rax adcq %rdx,%r12 movq %rbp,%rdx - adcq $0,%r10 + adcq $0,%r10 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r13 movq 24(%rbx),%rax - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r10,%r13 adcq %rbp,%r8 adcq $0,%r9 - +// ################################ * b[3] movq %rax,%rcx mulq 0(%rsi) addq %rax,%r11 @@ -350,15 +350,15 @@ adcq %rdx,%r9 adcq $0,%r10 - +// ################################ Last reduction step mulq 0(%r14) movq %r11,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq %r11,%rax adcq %rdx,%rcx subq %r11,%r13 - sbbq $0,%r11 + sbbq $0,%r11 // can't borrow mulq 8(%r14) addq %rcx,%r12 @@ -367,18 +367,18 @@ movq %rbp,%rax adcq %rdx,%r13 movq %rbp,%rdx - adcq $0,%r11 + adcq $0,%r11 // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r8 - sbbq %rdx,%rbp + sbbq %rdx,%rbp // can't borrow addq %r11,%r8 adcq %rbp,%r9 adcq $0,%r10 - +// ################################ Subtract ord movq %r12,%rsi subq 0(%r14),%r12 movq %r13,%r11 @@ -418,11 +418,11 @@ .cfi_endproc .size ecp_nistz256_ord_mul_mont_nohw,.-ecp_nistz256_ord_mul_mont_nohw - - - - - +// ############################################################################## +// void ecp_nistz256_ord_sqr_mont( +// uint64_t res[4], +// uint64_t a[4], +// uint64_t rep); .globl ecp_nistz256_ord_sqr_mont_nohw .hidden ecp_nistz256_ord_sqr_mont_nohw @@ -455,56 +455,56 @@ movq 8(%rsi),%rax movq 16(%rsi),%r14 movq 24(%rsi),%r15 - leaq .Lord(%rip),%rsi + leaq .Lord(%rip),%rsi // pointer to modulus movq %rdx,%rbx jmp .Loop_ord_sqr .align 32 .Loop_ord_sqr: - - movq %rax,%rbp - mulq %r8 +// ################################ a[1:] * a[0] + movq %rax,%rbp // put aside a[1] + mulq %r8 // a[1] * a[0] movq %rax,%r9 - movq %rbp,%xmm1 + movq %rbp,%xmm1 // offload a[1] movq %r14,%rax movq %rdx,%r10 - mulq %r8 + mulq %r8 // a[2] * a[0] addq %rax,%r10 movq %r15,%rax - movq %r14,%xmm2 + movq %r14,%xmm2 // offload a[2] adcq $0,%rdx movq %rdx,%r11 - mulq %r8 + mulq %r8 // a[3] * a[0] addq %rax,%r11 movq %r15,%rax - movq %r15,%xmm3 + movq %r15,%xmm3 // offload a[3] adcq $0,%rdx movq %rdx,%r12 - - mulq %r14 +// ################################ a[3] * a[2] + mulq %r14 // a[3] * a[2] movq %rax,%r13 movq %r14,%rax movq %rdx,%r14 - - mulq %rbp +// ################################ a[2:] * a[1] + mulq %rbp // a[2] * a[1] addq %rax,%r11 movq %r15,%rax adcq $0,%rdx movq %rdx,%r15 - mulq %rbp + mulq %rbp // a[3] * a[1] addq %rax,%r12 adcq $0,%rdx addq %r15,%r12 adcq %rdx,%r13 - adcq $0,%r14 + adcq $0,%r14 // can't overflow - +// ################################ *2 xorq %r15,%r15 movq %r8,%rax addq %r9,%r9 @@ -515,20 +515,20 @@ adcq %r14,%r14 adcq $0,%r15 - - mulq %rax +// ################################ Missing products + mulq %rax // a[0] * a[0] movq %rax,%r8 movq %xmm1,%rax movq %rdx,%rbp - mulq %rax + mulq %rax // a[1] * a[1] addq %rbp,%r9 adcq %rax,%r10 movq %xmm2,%rax adcq $0,%rdx movq %rdx,%rbp - mulq %rax + mulq %rax // a[2] * a[2] addq %rbp,%r11 adcq %rax,%r12 movq %xmm3,%rax @@ -536,23 +536,23 @@ movq %rdx,%rbp movq %r8,%rcx - imulq 32(%rsi),%r8 + imulq 32(%rsi),%r8 // *= .LordK - mulq %rax + mulq %rax // a[3] * a[3] addq %rbp,%r13 adcq %rax,%r14 - movq 0(%rsi),%rax - adcq %rdx,%r15 + movq 0(%rsi),%rax // modulus[0] + adcq %rdx,%r15 // can't overflow - +// ################################ First reduction step mulq %r8 movq %r8,%rbp - addq %rax,%rcx - movq 8(%rsi),%rax + addq %rax,%rcx // guaranteed to be zero + movq 8(%rsi),%rax // modulus[1] adcq %rdx,%rcx subq %r8,%r10 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r8 addq %rcx,%r9 @@ -561,29 +561,29 @@ movq %r8,%rax adcq %rdx,%r10 movq %r8,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow movq %r9,%rcx - imulq 32(%rsi),%r9 + imulq 32(%rsi),%r9 // *= .LordK shlq $32,%rax shrq $32,%rdx subq %rax,%r11 movq 0(%rsi),%rax - sbbq %rdx,%r8 + sbbq %rdx,%r8 // can't borrow addq %rbp,%r11 - adcq $0,%r8 + adcq $0,%r8 // can't overflow - +// ################################ Second reduction step mulq %r9 movq %r9,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq 8(%rsi),%rax adcq %rdx,%rcx subq %r9,%r11 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r9 addq %rcx,%r10 @@ -592,29 +592,29 @@ movq %r9,%rax adcq %rdx,%r11 movq %r9,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow movq %r10,%rcx - imulq 32(%rsi),%r10 + imulq 32(%rsi),%r10 // *= .LordK shlq $32,%rax shrq $32,%rdx subq %rax,%r8 movq 0(%rsi),%rax - sbbq %rdx,%r9 + sbbq %rdx,%r9 // can't borrow addq %rbp,%r8 - adcq $0,%r9 + adcq $0,%r9 // can't overflow - +// ################################ Third reduction step mulq %r10 movq %r10,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq 8(%rsi),%rax adcq %rdx,%rcx subq %r10,%r8 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r10 addq %rcx,%r11 @@ -623,29 +623,29 @@ movq %r10,%rax adcq %rdx,%r8 movq %r10,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow movq %r11,%rcx - imulq 32(%rsi),%r11 + imulq 32(%rsi),%r11 // *= .LordK shlq $32,%rax shrq $32,%rdx subq %rax,%r9 movq 0(%rsi),%rax - sbbq %rdx,%r10 + sbbq %rdx,%r10 // can't borrow addq %rbp,%r9 - adcq $0,%r10 + adcq $0,%r10 // can't overflow - +// ################################ Last reduction step mulq %r11 movq %r11,%rbp - addq %rax,%rcx + addq %rax,%rcx // guaranteed to be zero movq 8(%rsi),%rax adcq %rdx,%rcx subq %r11,%r9 - sbbq $0,%rbp + sbbq $0,%rbp // can't borrow mulq %r11 addq %rcx,%r8 @@ -654,17 +654,17 @@ movq %r11,%rax adcq %rdx,%r9 movq %r11,%rdx - adcq $0,%rbp + adcq $0,%rbp // can't overflow shlq $32,%rax shrq $32,%rdx subq %rax,%r10 - sbbq %rdx,%r11 + sbbq %rdx,%r11 // can't borrow addq %rbp,%r10 - adcq $0,%r11 + adcq $0,%r11 // can't overflow - +// ################################ Add bits [511:256] of the sqr result xorq %rdx,%rdx addq %r12,%r8 adcq %r13,%r9 @@ -674,7 +674,7 @@ movq %r9,%rax adcq $0,%rdx - +// ################################ Compare to modulus subq 0(%rsi),%r8 movq %r10,%r14 sbbq 8(%rsi),%r9 @@ -717,7 +717,7 @@ ret .cfi_endproc .size ecp_nistz256_ord_sqr_mont_nohw,.-ecp_nistz256_ord_sqr_mont_nohw - +// ############################################################################## .globl ecp_nistz256_ord_mul_mont_adx .hidden ecp_nistz256_ord_mul_mont_adx .type ecp_nistz256_ord_mul_mont_adx,@function @@ -752,11 +752,11 @@ movq 8(%rsi),%r10 movq 16(%rsi),%r11 movq 24(%rsi),%r12 - leaq -128(%rsi),%rsi + leaq -128(%rsi),%rsi // control u-op density leaq .Lord-128(%rip),%r14 movq .LordK(%rip),%r15 - +// ################################ Multiply by b[0] mulxq %r9,%r8,%r9 mulxq %r10,%rcx,%r10 mulxq %r11,%rbp,%r11 @@ -768,10 +768,10 @@ adcq %rcx,%r11 adcq $0,%r12 - - xorq %r13,%r13 +// ################################ reduction + xorq %r13,%r13 // %r13=0, cf=0, of=0 mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r8 + adcxq %rcx,%r8 // guaranteed to be zero adoxq %rbp,%r9 mulxq 8+128(%r14),%rcx,%rbp @@ -788,9 +788,9 @@ adoxq %rbp,%r12 adcxq %r8,%r12 adoxq %r8,%r13 - adcq $0,%r13 + adcq $0,%r13 // cf=0, of=0 - +// ################################ Multiply by b[1] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r9 adoxq %rbp,%r10 @@ -811,11 +811,11 @@ adcxq %r8,%r13 adoxq %r8,%r8 - adcq $0,%r8 + adcq $0,%r8 // cf=0, of=0 - +// ################################ reduction mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r9 + adcxq %rcx,%r9 // guaranteed to be zero adoxq %rbp,%r10 mulxq 8+128(%r14),%rcx,%rbp @@ -832,9 +832,9 @@ adoxq %rbp,%r13 adcxq %r9,%r13 adoxq %r9,%r8 - adcq $0,%r8 + adcq $0,%r8 // cf=0, of=0 - +// ################################ Multiply by b[2] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r10 adoxq %rbp,%r11 @@ -855,11 +855,11 @@ adcxq %r9,%r8 adoxq %r9,%r9 - adcq $0,%r9 + adcq $0,%r9 // cf=0, of=0 - +// ################################ reduction mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r10 + adcxq %rcx,%r10 // guaranteed to be zero adoxq %rbp,%r11 mulxq 8+128(%r14),%rcx,%rbp @@ -876,9 +876,9 @@ adoxq %rbp,%r8 adcxq %r10,%r8 adoxq %r10,%r9 - adcq $0,%r9 + adcq $0,%r9 // cf=0, of=0 - +// ################################ Multiply by b[3] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r11 adoxq %rbp,%r12 @@ -899,11 +899,11 @@ adcxq %r10,%r9 adoxq %r10,%r10 - adcq $0,%r10 + adcq $0,%r10 // cf=0, of=0 - +// ################################ reduction mulxq 0+128(%r14),%rcx,%rbp - adcxq %rcx,%r11 + adcxq %rcx,%r11 // guaranteed to be zero adoxq %rbp,%r12 mulxq 8+128(%r14),%rcx,%rbp @@ -924,8 +924,8 @@ adoxq %r11,%r10 adcq $0,%r10 - - +// ################################ +// Branch-less conditional subtraction of P movq %r8,%rcx subq 0(%r14),%r12 sbbq 8(%r14),%r13 @@ -1001,38 +1001,38 @@ .align 32 .Loop_ord_sqrx: - mulxq %r14,%r9,%r10 - mulxq %r15,%rcx,%r11 - movq %rdx,%rax - movq %r14,%xmm1 - mulxq %r8,%rbp,%r12 + mulxq %r14,%r9,%r10 // a[0]*a[1] + mulxq %r15,%rcx,%r11 // a[0]*a[2] + movq %rdx,%rax // offload a[0] + movq %r14,%xmm1 // offload a[1] + mulxq %r8,%rbp,%r12 // a[0]*a[3] movq %r14,%rdx addq %rcx,%r10 - movq %r15,%xmm2 + movq %r15,%xmm2 // offload a[2] adcq %rbp,%r11 adcq $0,%r12 - xorq %r13,%r13 - - mulxq %r15,%rcx,%rbp + xorq %r13,%r13 // %r13=0,cf=0,of=0 +// ################################ + mulxq %r15,%rcx,%rbp // a[1]*a[2] adcxq %rcx,%r11 adoxq %rbp,%r12 - mulxq %r8,%rcx,%rbp + mulxq %r8,%rcx,%rbp // a[1]*a[3] movq %r15,%rdx adcxq %rcx,%r12 adoxq %rbp,%r13 adcq $0,%r13 - - mulxq %r8,%rcx,%r14 +// ################################ + mulxq %r8,%rcx,%r14 // a[2]*a[3] movq %rax,%rdx - movq %r8,%xmm3 - xorq %r15,%r15 - adcxq %r9,%r9 + movq %r8,%xmm3 // offload a[3] + xorq %r15,%r15 // %r15=0,cf=0,of=0 + adcxq %r9,%r9 // acc1:6<<1 adoxq %rcx,%r13 adcxq %r10,%r10 - adoxq %r15,%r14 + adoxq %r15,%r14 // of=0 - +// ################################ a[i]*a[i] mulxq %rdx,%r8,%rbp movq %xmm1,%rdx adcxq %r11,%r11 @@ -1054,13 +1054,13 @@ adoxq %rcx,%r14 adoxq %rax,%r15 - +// ################################ reduction movq %r8,%rdx mulxq 32(%rsi),%rdx,%rcx - xorq %rax,%rax + xorq %rax,%rax // cf=0, of=0 mulxq 0(%rsi),%rcx,%rbp - adcxq %rcx,%r8 + adcxq %rcx,%r8 // guaranteed to be zero adoxq %rbp,%r9 mulxq 8(%rsi),%rcx,%rbp adcxq %rcx,%r9 @@ -1070,15 +1070,15 @@ adoxq %rbp,%r11 mulxq 24(%rsi),%rcx,%rbp adcxq %rcx,%r11 - adoxq %rbp,%r8 - adcxq %rax,%r8 + adoxq %rbp,%r8 // of=0 + adcxq %rax,%r8 // cf=0 - +// ################################ movq %r9,%rdx mulxq 32(%rsi),%rdx,%rcx mulxq 0(%rsi),%rcx,%rbp - adoxq %rcx,%r9 + adoxq %rcx,%r9 // guaranteed to be zero adcxq %rbp,%r10 mulxq 8(%rsi),%rcx,%rbp adoxq %rcx,%r10 @@ -1088,15 +1088,15 @@ adcxq %rbp,%r8 mulxq 24(%rsi),%rcx,%rbp adoxq %rcx,%r8 - adcxq %rbp,%r9 - adoxq %rax,%r9 + adcxq %rbp,%r9 // cf=0 + adoxq %rax,%r9 // of=0 - +// ################################ movq %r10,%rdx mulxq 32(%rsi),%rdx,%rcx mulxq 0(%rsi),%rcx,%rbp - adcxq %rcx,%r10 + adcxq %rcx,%r10 // guaranteed to be zero adoxq %rbp,%r11 mulxq 8(%rsi),%rcx,%rbp adcxq %rcx,%r11 @@ -1106,15 +1106,15 @@ adoxq %rbp,%r9 mulxq 24(%rsi),%rcx,%rbp adcxq %rcx,%r9 - adoxq %rbp,%r10 - adcxq %rax,%r10 + adoxq %rbp,%r10 // of=0 + adcxq %rax,%r10 // cf=0 - +// ################################ movq %r11,%rdx mulxq 32(%rsi),%rdx,%rcx mulxq 0(%rsi),%rcx,%rbp - adoxq %rcx,%r11 + adoxq %rcx,%r11 // guaranteed to be zero adcxq %rbp,%r8 mulxq 8(%rsi),%rcx,%rbp adoxq %rcx,%r8 @@ -1127,8 +1127,8 @@ adcxq %rbp,%r11 adoxq %rax,%r11 - - addq %r8,%r12 +// ################################ accumulate upper half + addq %r8,%r12 // add %r12, %r8 adcq %r13,%r9 movq %r12,%rdx adcq %r14,%r10 @@ -1136,7 +1136,7 @@ movq %r9,%r14 adcq $0,%rax - +// ################################ compare to modulus subq 0(%rsi),%r12 movq %r10,%r15 sbbq 8(%rsi),%r9 @@ -1179,11 +1179,11 @@ ret .cfi_endproc .size ecp_nistz256_ord_sqr_mont_adx,.-ecp_nistz256_ord_sqr_mont_adx - - - - - +// ############################################################################## +// void ecp_nistz256_mul_mont( +// uint64_t res[4], +// uint64_t a[4], +// uint64_t b[4]); .globl ecp_nistz256_mul_mont_nohw .hidden ecp_nistz256_mul_mont_nohw @@ -1243,8 +1243,8 @@ .align 32 __ecp_nistz256_mul_montq: .cfi_startproc - - +// ####################################################################### +// Multiply a by b[0] movq %rax,%rbp mulq %r9 movq .Lpoly+8(%rip),%r14 @@ -1272,20 +1272,20 @@ xorq %r13,%r13 movq %rdx,%r12 - - - - - - - - +// ####################################################################### +// First reduction step +// Basically now we want to multiply acc[0] by p256, +// and add the result to the acc. +// Due to the special form of p256 we do some optimizations +// +// acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0] +// then we add acc[0] and get acc[0] x 2^96 movq %r8,%rbp shlq $32,%r8 mulq %r15 shrq $32,%rbp - addq %r8,%r9 + addq %r8,%r9 // +=acc[0]<<96 adcq %rbp,%r10 adcq %rax,%r11 movq 8(%rbx),%rax @@ -1293,8 +1293,8 @@ adcq $0,%r13 xorq %r8,%r8 - - +// ####################################################################### +// Multiply by b[1] movq %rax,%rbp mulq 0(%rsi) addq %rax,%r9 @@ -1326,8 +1326,8 @@ adcq %rdx,%r13 adcq $0,%r8 - - +// ####################################################################### +// Second reduction step movq %r9,%rbp shlq $32,%r9 mulq %r15 @@ -1340,8 +1340,8 @@ adcq $0,%r8 xorq %r9,%r9 - - +// ####################################################################### +// Multiply by b[2] movq %rax,%rbp mulq 0(%rsi) addq %rax,%r10 @@ -1373,8 +1373,8 @@ adcq %rdx,%r8 adcq $0,%r9 - - +// ####################################################################### +// Third reduction step movq %r10,%rbp shlq $32,%r10 mulq %r15 @@ -1387,8 +1387,8 @@ adcq $0,%r9 xorq %r10,%r10 - - +// ####################################################################### +// Multiply by b[3] movq %rax,%rbp mulq 0(%rsi) addq %rax,%r11 @@ -1420,8 +1420,8 @@ adcq %rdx,%r9 adcq $0,%r10 - - +// ####################################################################### +// Final reduction step movq %r11,%rbp shlq $32,%r11 mulq %r15 @@ -1434,14 +1434,14 @@ movq %r13,%rbp adcq $0,%r10 - - - subq $-1,%r12 +// ####################################################################### +// Branch-less conditional subtraction of P + subq $-1,%r12 // .Lpoly[0] movq %r8,%rbx - sbbq %r14,%r13 - sbbq $0,%r8 + sbbq %r14,%r13 // .Lpoly[1] + sbbq $0,%r8 // .Lpoly[2] movq %r9,%rdx - sbbq %r15,%r9 + sbbq %r15,%r9 // .Lpoly[3] sbbq $0,%r10 cmovcq %rcx,%r12 @@ -1457,13 +1457,13 @@ .cfi_endproc .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq +// ############################################################################## +// void ecp_nistz256_sqr_mont( +// uint64_t res[4], +// uint64_t a[4]); - - - - - - +// we optimize the square according to S.Gueron and V.Krasnov, +// "Speeding up Big-Number Squaring" .globl ecp_nistz256_sqr_mont_nohw .hidden ecp_nistz256_sqr_mont_nohw .type ecp_nistz256_sqr_mont_nohw,@function @@ -1521,31 +1521,31 @@ __ecp_nistz256_sqr_montq: .cfi_startproc movq %rax,%r13 - mulq %r14 + mulq %r14 // a[1]*a[0] movq %rax,%r9 movq %r15,%rax movq %rdx,%r10 - mulq %r13 + mulq %r13 // a[0]*a[2] addq %rax,%r10 movq %r8,%rax adcq $0,%rdx movq %rdx,%r11 - mulq %r13 + mulq %r13 // a[0]*a[3] addq %rax,%r11 movq %r15,%rax adcq $0,%rdx movq %rdx,%r12 - - mulq %r14 +// ################################ + mulq %r14 // a[1]*a[2] addq %rax,%r11 movq %r8,%rax adcq $0,%rdx movq %rdx,%rbp - mulq %r14 + mulq %r14 // a[1]*a[3] addq %rax,%r12 movq %r8,%rax adcq $0,%rdx @@ -1553,15 +1553,15 @@ movq %rdx,%r13 adcq $0,%r13 - - mulq %r15 +// ################################ + mulq %r15 // a[2]*a[3] xorq %r15,%r15 addq %rax,%r13 movq 0(%rsi),%rax movq %rdx,%r14 adcq $0,%r14 - addq %r9,%r9 + addq %r9,%r9 // acc1:6<<1 adcq %r10,%r10 adcq %r11,%r11 adcq %r12,%r12 @@ -1597,21 +1597,21 @@ movq .Lpoly+8(%rip),%rsi movq .Lpoly+24(%rip),%rbp - - - +// ######################################### +// Now the reduction +// First iteration movq %r8,%rcx shlq $32,%r8 mulq %rbp shrq $32,%rcx - addq %r8,%r9 + addq %r8,%r9 // +=acc[0]<<96 adcq %rcx,%r10 adcq %rax,%r11 movq %r9,%rax adcq $0,%rdx - - +// ######################################### +// Second iteration movq %r9,%rcx shlq $32,%r9 movq %rdx,%r8 @@ -1623,8 +1623,8 @@ movq %r10,%rax adcq $0,%rdx - - +// ######################################### +// Third iteration movq %r10,%rcx shlq $32,%r10 movq %rdx,%r9 @@ -1636,8 +1636,8 @@ movq %r11,%rax adcq $0,%rdx - - +// ########################################## +// Last iteration movq %r11,%rcx shlq $32,%r11 movq %rdx,%r10 @@ -1649,8 +1649,8 @@ adcq $0,%rdx xorq %r11,%r11 - - +// ########################################### +// Add the rest of the acc addq %r8,%r12 adcq %r9,%r13 movq %r12,%r8 @@ -1659,12 +1659,12 @@ movq %r13,%r9 adcq $0,%r11 - subq $-1,%r12 + subq $-1,%r12 // .Lpoly[0] movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 + sbbq %rsi,%r13 // .Lpoly[1] + sbbq $0,%r14 // .Lpoly[2] movq %r15,%rcx - sbbq %rbp,%r15 + sbbq %rbp,%r15 // .Lpoly[3] sbbq $0,%r11 cmovcq %r8,%r12 @@ -1711,7 +1711,7 @@ movq 8(%rsi),%r10 movq 16(%rsi),%r11 movq 24(%rsi),%r12 - leaq -128(%rsi),%rsi + leaq -128(%rsi),%rsi // control u-op density call __ecp_nistz256_mul_montx @@ -1738,12 +1738,12 @@ .align 32 __ecp_nistz256_mul_montx: .cfi_startproc - - +// ####################################################################### +// Multiply by b[0] mulxq %r9,%r8,%r9 mulxq %r10,%rcx,%r10 movq $32,%r14 - xorq %r13,%r13 + xorq %r13,%r13 // cf=0 mulxq %r11,%rbp,%r11 movq .Lpoly+24(%rip),%r15 adcq %rcx,%r9 @@ -1755,8 +1755,8 @@ shrxq %r14,%r8,%rcx adcq $0,%r12 - - +// ####################################################################### +// First reduction step addq %rbp,%r9 adcq %rcx,%r10 @@ -1765,10 +1765,10 @@ adcq %rcx,%r11 adcq %rbp,%r12 adcq $0,%r13 - xorq %r8,%r8 + xorq %r8,%r8 // %r8=0,cf=0,of=0 - - +// ####################################################################### +// Multiply by b[1] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r9 adoxq %rbp,%r10 @@ -1792,8 +1792,8 @@ adoxq %r8,%r8 adcq $0,%r8 - - +// ####################################################################### +// Second reduction step addq %rcx,%r10 adcq %rbp,%r11 @@ -1802,10 +1802,10 @@ adcq %rcx,%r12 adcq %rbp,%r13 adcq $0,%r8 - xorq %r9,%r9 + xorq %r9,%r9 // %r9=0,cf=0,of=0 - - +// ####################################################################### +// Multiply by b[2] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r10 adoxq %rbp,%r11 @@ -1829,8 +1829,8 @@ adoxq %r9,%r9 adcq $0,%r9 - - +// ####################################################################### +// Third reduction step addq %rcx,%r11 adcq %rbp,%r12 @@ -1839,10 +1839,10 @@ adcq %rcx,%r13 adcq %rbp,%r8 adcq $0,%r9 - xorq %r10,%r10 + xorq %r10,%r10 // %r10=0,cf=0,of=0 - - +// ####################################################################### +// Multiply by b[3] mulxq 0+128(%rsi),%rcx,%rbp adcxq %rcx,%r11 adoxq %rbp,%r12 @@ -1866,8 +1866,8 @@ adoxq %r10,%r10 adcq $0,%r10 - - +// ####################################################################### +// Fourth reduction step addq %rcx,%r12 adcq %rbp,%r13 @@ -1879,15 +1879,15 @@ adcq %rbp,%r9 adcq $0,%r10 - - +// ####################################################################### +// Branch-less conditional subtraction of P xorl %eax,%eax movq %r8,%rcx - sbbq $-1,%r12 - sbbq %r14,%r13 - sbbq $0,%r8 + sbbq $-1,%r12 // .Lpoly[0] + sbbq %r14,%r13 // .Lpoly[1] + sbbq $0,%r8 // .Lpoly[2] movq %r9,%rbp - sbbq %r15,%r9 + sbbq %r15,%r9 // .Lpoly[3] sbbq $0,%r10 cmovcq %rbx,%r12 @@ -1933,7 +1933,7 @@ movq 8(%rsi),%r14 movq 16(%rsi),%r15 movq 24(%rsi),%r8 - leaq -128(%rsi),%rsi + leaq -128(%rsi),%rsi // control u-op density call __ecp_nistz256_sqr_montx @@ -1960,35 +1960,35 @@ .align 32 __ecp_nistz256_sqr_montx: .cfi_startproc - mulxq %r14,%r9,%r10 - mulxq %r15,%rcx,%r11 + mulxq %r14,%r9,%r10 // a[0]*a[1] + mulxq %r15,%rcx,%r11 // a[0]*a[2] xorl %eax,%eax adcq %rcx,%r10 - mulxq %r8,%rbp,%r12 + mulxq %r8,%rbp,%r12 // a[0]*a[3] movq %r14,%rdx adcq %rbp,%r11 adcq $0,%r12 - xorq %r13,%r13 + xorq %r13,%r13 // %r13=0,cf=0,of=0 - - mulxq %r15,%rcx,%rbp +// ################################ + mulxq %r15,%rcx,%rbp // a[1]*a[2] adcxq %rcx,%r11 adoxq %rbp,%r12 - mulxq %r8,%rcx,%rbp + mulxq %r8,%rcx,%rbp // a[1]*a[3] movq %r15,%rdx adcxq %rcx,%r12 adoxq %rbp,%r13 adcq $0,%r13 - - mulxq %r8,%rcx,%r14 +// ################################ + mulxq %r8,%rcx,%r14 // a[2]*a[3] movq 0+128(%rsi),%rdx - xorq %r15,%r15 - adcxq %r9,%r9 + xorq %r15,%r15 // %r15=0,cf=0,of=0 + adcxq %r9,%r9 // acc1:6<<1 adoxq %rcx,%r13 adcxq %r10,%r10 - adoxq %r15,%r14 + adoxq %r15,%r14 // of=0 mulxq %rdx,%r8,%rbp movq 8+128(%rsi),%rdx @@ -2017,7 +2017,7 @@ shrxq %rsi,%r8,%rax movq %rdx,%rbp - +// reduction step 1 addq %rcx,%r9 adcq %rax,%r10 @@ -2027,7 +2027,7 @@ adcq $0,%r8 shrxq %rsi,%r9,%rax - +// reduction step 2 addq %rcx,%r10 adcq %rax,%r11 @@ -2037,7 +2037,7 @@ adcq $0,%r9 shrxq %rsi,%r10,%rax - +// reduction step 3 addq %rcx,%r11 adcq %rax,%r8 @@ -2047,7 +2047,7 @@ adcq $0,%r10 shrxq %rsi,%r11,%rax - +// reduction step 4 addq %rcx,%r8 adcq %rax,%r9 @@ -2056,7 +2056,7 @@ adcq $0,%r11 xorq %rdx,%rdx - addq %r8,%r12 + addq %r8,%r12 // accumulate upper half movq .Lpoly+8(%rip),%rsi adcq %r9,%r13 movq %r12,%r8 @@ -2065,12 +2065,12 @@ movq %r13,%r9 adcq $0,%rdx - subq $-1,%r12 + subq $-1,%r12 // .Lpoly[0] movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 + sbbq %rsi,%r13 // .Lpoly[1] + sbbq $0,%r14 // .Lpoly[2] movq %r15,%r11 - sbbq %rbp,%r15 + sbbq %rbp,%r15 // .Lpoly[3] sbbq $0,%rdx cmovcq %r8,%r12 @@ -2085,8 +2085,8 @@ ret .cfi_endproc .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx - - +// ############################################################################## +// void ecp_nistz256_select_w5_nohw(uint64_t *val, uint64_t *in_t, int index); .globl ecp_nistz256_select_w5_nohw .hidden ecp_nistz256_select_w5_nohw .type ecp_nistz256_select_w5_nohw,@function @@ -2149,8 +2149,8 @@ .LSEH_end_ecp_nistz256_select_w5_nohw: .size ecp_nistz256_select_w5_nohw,.-ecp_nistz256_select_w5_nohw - - +// ############################################################################## +// void ecp_nistz256_select_w7_nohw(uint64_t *val, uint64_t *in_t, int index); .globl ecp_nistz256_select_w7_nohw .hidden ecp_nistz256_select_w7_nohw .type ecp_nistz256_select_w7_nohw,@function @@ -2201,8 +2201,8 @@ .cfi_endproc .LSEH_end_ecp_nistz256_select_w7_nohw: .size ecp_nistz256_select_w7_nohw,.-ecp_nistz256_select_w7_nohw - - +// ############################################################################## +// void ecp_nistz256_select_w5_avx2(uint64_t *val, uint64_t *in_t, int index); .globl ecp_nistz256_select_w5_avx2 .hidden ecp_nistz256_select_w5_avx2 .type ecp_nistz256_select_w5_avx2,@function @@ -2267,8 +2267,8 @@ .LSEH_end_ecp_nistz256_select_w5_avx2: .size ecp_nistz256_select_w5_avx2,.-ecp_nistz256_select_w5_avx2 - - +// ############################################################################## +// void ecp_nistz256_select_w7_avx2(uint64_t *val, uint64_t *in_t, int index); .globl ecp_nistz256_select_w7_avx2 .hidden ecp_nistz256_select_w7_avx2 .type ecp_nistz256_select_w7_avx2,@function @@ -2288,7 +2288,7 @@ vmovd %edx,%xmm1 vpermd %ymm1,%ymm2,%ymm1 - +// Skip index = 0, because it is implicitly the point at infinity movq $21,%rax .Lselect_loop_avx2_w7: @@ -2448,7 +2448,7 @@ __ecp_nistz256_mul_by_2q: .cfi_startproc xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 adcq %r13,%r13 movq %r12,%rax adcq %r8,%r8 @@ -2506,10 +2506,10 @@ .Lpoint_doubleq_body: .Lpoint_double_shortcutq: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi.x + movq %rsi,%rbx // backup copy movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 + movq 32+0(%rsi),%r12 // load in_y in "5-4-0-1" order movq 32+8(%rsi),%r13 movq 32+16(%rsi),%r8 movq 32+24(%rsi),%r9 @@ -2524,7 +2524,7 @@ movq %r11,%xmm2 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_by_2q // p256_mul_by_2(S, in_y); movq 64+0(%rsi),%rax movq 64+8(%rsi),%r14 @@ -2532,7 +2532,7 @@ movq 64+24(%rsi),%r8 leaq 64-0(%rsi),%rsi leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Zsqr, in_z); movq 0+0(%rsp),%rax movq 8+0(%rsp),%r14 @@ -2540,9 +2540,9 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(S, S); - movq 32(%rbx),%rax + movq 32(%rbx),%rax // %rbx is still valid movq 64+0(%rbx),%r9 movq 64+8(%rbx),%r10 movq 64+16(%rbx),%r11 @@ -2550,24 +2550,24 @@ leaq 64-0(%rbx),%rsi leaq 32(%rbx),%rbx movq %xmm2,%rdi - call __ecp_nistz256_mul_montq - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2q // p256_mul_by_2(res_z, res_z); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq + call __ecp_nistz256_add_toq // p256_add(M, in_x, Zsqr); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(Zsqr, in_x, Zsqr); movq 0+0(%rsp),%rax movq 8+0(%rsp),%r14 @@ -2575,7 +2575,7 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 movq %xmm1,%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(res_y, S); xorq %r9,%r9 movq %r12,%rax addq $-1,%r12 @@ -2586,7 +2586,7 @@ movq %r15,%r8 adcq %rbp,%r15 adcq $0,%r9 - xorq %rsi,%rsi + xorq %rsi,%rsi // borrow %rsi testq $1,%rax cmovzq %rax,%r12 @@ -2595,7 +2595,7 @@ cmovzq %r8,%r15 cmovzq %rsi,%r9 - movq %r13,%rax + movq %r13,%rax // a0:a3>>1 shrq $1,%r12 shlq $63,%rax movq %r14,%r10 @@ -2622,14 +2622,14 @@ movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(M, M, Zsqr); leaq 128(%rsp),%rdi call __ecp_nistz256_mul_by_2q leaq 32(%rsp),%rbx leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq + call __ecp_nistz256_add_toq // p256_mul_by_3(M, M); movq 96(%rsp),%rax leaq 96(%rsp),%rbx @@ -2639,10 +2639,10 @@ movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S, S, in_x); leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_by_2q // p256_mul_by_2(tmp0, S); movq 0+32(%rsp),%rax movq 8+32(%rsp),%r14 @@ -2650,27 +2650,27 @@ movq 16+32(%rsp),%r15 movq 24+32(%rsp),%r8 movq %xmm0,%rdi - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq // p256_sqr_mont(res_x, M); leaq 128(%rsp),%rbx - movq %r14,%r8 + movq %r14,%r8 // harmonize sqr output and sub input movq %r15,%r9 movq %rsi,%r14 movq %rbp,%r15 - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_x, res_x, tmp0); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 + movq 0+24(%rsp),%r10 // "4-5-0-1" order leaq 0(%rsp),%rdi - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(S, S, res_x); movq 32(%rsp),%rax leaq 32(%rsp),%rbx - movq %r12,%r14 + movq %r12,%r14 // harmonize sub output and mul input xorl %ecx,%ecx - movq %r12,0+0(%rsp) + movq %r12,0+0(%rsp) // have to save:-( movq %r13,%r10 movq %r13,0+8(%rsp) cmovzq %r8,%r11 @@ -2680,11 +2680,11 @@ movq %r9,0+24(%rsp) movq %r14,%r9 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S, S, M); movq %xmm1,%rbx movq %xmm1,%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_y, S, res_y); leaq 160+56(%rsp),%rsi .cfi_def_cfa %rsi,8 @@ -2735,14 +2735,14 @@ .cfi_adjust_cfa_offset 32*18+8 .Lpoint_addq_body: - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi + movq %rsi,%rbx // reassign + movq %rdx,%rsi // reassign movdqa %xmm0,384(%rsp) movdqa %xmm1,384+16(%rsp) movdqa %xmm2,416(%rsp) @@ -2751,54 +2751,54 @@ movdqa %xmm5,448+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 por %xmm3,%xmm5 movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rax + movq 64+0(%rsi),%rax // load original in2_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 movdqa %xmm0,480(%rsp) pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 + movdqu 64(%rsi),%xmm0 // in2_z again movdqu 80(%rsi),%xmm1 movdqa %xmm2,512(%rsp) movdqa %xmm3,512+16(%rsp) por %xmm4,%xmm5 pxor %xmm4,%xmm4 por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi - leaq 64-0(%rsi),%rsi - movq %rax,544+0(%rsp) + leaq 64-0(%rsi),%rsi // %rsi is still valid + movq %rax,544+0(%rsp) // make in2_z copy movq %r14,544+8(%rsp) movq %r15,544+16(%rsp) movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 96(%rsp),%rdi // Z2^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Z2sqr, in2_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm1,%xmm4 por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 por %xmm3,%xmm4 pxor %xmm3,%xmm3 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rax + pshufd $0,%xmm4,%xmm4 // in2infty + movq 64+0(%rbx),%rax // load original in1_z movq 64+8(%rbx),%r14 movq 64+16(%rbx),%r15 movq 64+24(%rbx),%r8 movq %rbx,%xmm1 leaq 64-0(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Z1sqr, in1_z); movq 544(%rsp),%rax leaq 544(%rsp),%rbx @@ -2807,8 +2807,8 @@ leaq 0+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 224(%rsp),%rdi // S1 = Z2^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S1, Z2sqr, in2_z); movq 448(%rsp),%rax leaq 448(%rsp),%rbx @@ -2817,8 +2817,8 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 256(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, Z1sqr, in1_z); movq 416(%rsp),%rax leaq 416(%rsp),%rbx @@ -2827,8 +2827,8 @@ leaq 0+224(%rsp),%rsi movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 224(%rsp),%rdi // S1 = Y1*Z2^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S1, S1, in1_y); movq 512(%rsp),%rax leaq 512(%rsp),%rbx @@ -2837,18 +2837,18 @@ leaq 0+256(%rsp),%rsi movq 16+256(%rsp),%r11 movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 256(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, S2, in2_y); leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 64(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromq // p256_sub(R, S2, S1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero movdqa %xmm4,%xmm2 orq %r8,%r12 orq %r9,%r12 - por %xmm5,%xmm2 + por %xmm5,%xmm2 // in1infty || in2infty movq %r12,%xmm3 movq 384(%rsp),%rax @@ -2858,8 +2858,8 @@ leaq 0+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 160(%rsp),%rdi // U1 = X1*Z2^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U1, in1_x, Z2sqr); movq 480(%rsp),%rax leaq 480(%rsp),%rbx @@ -2868,34 +2868,34 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 192(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, in2_x, Z1sqr); leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 0(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromq // p256_sub(H, U2, U1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero orq %r8,%r12 - orq %r9,%r12 + orq %r9,%r12 // movq %xmm2,%r8 movq %xmm3,%r9 orq %r8,%r12 -.byte 0x3e - jnz .Ladd_proceedq +.byte 0x3e // predict taken + jnz .Ladd_proceedq // - - +// We now know A = B or A = -B and neither is infinity. Compare the +// y-coordinates via S1 and S2. testq %r9,%r9 - jz .Ladd_doubleq + jz .Ladd_doubleq // is_equal(S1, S2) - - - - - - movq %xmm0,%rdi +// A = -B, so the result is infinity. +// +// TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in +// which case we should eliminate this special-case and simplify the +// timing analysis. + movq %xmm0,%rdi // restore %rdi pxor %xmm0,%xmm0 movdqu %xmm0,0(%rdi) movdqu %xmm0,16(%rdi) @@ -2907,9 +2907,9 @@ .align 32 .Ladd_doubleq: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp + movq %xmm1,%rsi // restore %rsi + movq %xmm0,%rdi // restore %rdi + addq $416,%rsp // difference in frame sizes .cfi_adjust_cfa_offset -416 jmp .Lpoint_double_shortcutq .cfi_adjust_cfa_offset 416 @@ -2921,8 +2921,8 @@ leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 96(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Rsqr, R); movq 448(%rsp),%rax leaq 448(%rsp),%rbx @@ -2931,16 +2931,16 @@ leaq 0+0(%rsp),%rsi movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, H, in1_z); movq 0+0(%rsp),%rax movq 8+0(%rsp),%r14 leaq 0+0(%rsp),%rsi movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 32(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Hsqr, H); movq 544(%rsp),%rax leaq 544(%rsp),%rbx @@ -2949,8 +2949,8 @@ leaq 0+352(%rsp),%rsi movq 16+352(%rsp),%r11 movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, res_z, in2_z); movq 0(%rsp),%rax leaq 0(%rsp),%rbx @@ -2959,8 +2959,8 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 128(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(Hcub, Hsqr, H); movq 160(%rsp),%rax leaq 160(%rsp),%rbx @@ -2969,14 +2969,14 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - + leaq 192(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, U1, Hsqr); +// lea 192(%rsp), %rsi +// lea 32(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 96(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -3002,11 +3002,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(res_x, Rsqr, Hsqr); leaq 128(%rsp),%rbx leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_x, res_x, Hcub); movq 192+0(%rsp),%rax movq 192+8(%rsp),%rbp @@ -3014,10 +3014,10 @@ movq 192+24(%rsp),%r10 leaq 320(%rsp),%rdi - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(res_y, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 128(%rsp),%rax @@ -3028,7 +3028,7 @@ movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, S1, Hcub); movq 320(%rsp),%rax leaq 320(%rsp),%rbx @@ -3038,15 +3038,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(res_y, R, res_y); leaq 256(%rsp),%rbx leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_y, res_y, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, in2_z, in1infty); movdqa %xmm5,%xmm1 pandn 352(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3057,7 +3057,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3070,7 +3070,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3081,7 +3081,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3094,7 +3094,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 320(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3105,7 +3105,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3168,14 +3168,14 @@ .cfi_adjust_cfa_offset 32*15+8 .Ladd_affineq_body: - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi + movq %rdx,%rbx // reassign movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rax + movq 64+0(%rsi),%rax // load original in1_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 @@ -3187,7 +3187,7 @@ movdqa %xmm5,384+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rbx),%xmm0 + movdqu 0(%rbx),%xmm0 // copy *(P256_POINT_AFFINE *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rbx),%xmm1 movdqu 32(%rbx),%xmm2 @@ -3197,7 +3197,7 @@ pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,416+16(%rsp) por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi movdqa %xmm2,448(%rsp) movdqa %xmm3,448+16(%rsp) por %xmm2,%xmm3 @@ -3205,33 +3205,33 @@ pxor %xmm4,%xmm4 por %xmm1,%xmm3 - leaq 64-0(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 64-0(%rsi),%rsi // %rsi is still valid + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Z1sqr, in1_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rax - - movq %r12,%r9 + movq 0(%rbx),%rax // %rbx is still valid +// lea 0x00(%rbx), %rbx + movq %r12,%r9 // harmonize sqr output and mul input por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 movq %r13,%r10 por %xmm3,%xmm4 pxor %xmm3,%xmm3 movq %r14,%r11 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 + pshufd $0,%xmm4,%xmm4 // in2infty leaq 32-0(%rsp),%rsi movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 0(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, Z1sqr, in2_x); leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 64(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromq // p256_sub(H, U2, in1_x); movq 384(%rsp),%rax leaq 384(%rsp),%rbx @@ -3240,8 +3240,8 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 32(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, Z1sqr, in1_z); movq 384(%rsp),%rax leaq 384(%rsp),%rbx @@ -3250,8 +3250,8 @@ leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 288(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq // p256_mul_mont(res_z, H, in1_z); movq 448(%rsp),%rax leaq 448(%rsp),%rbx @@ -3260,28 +3260,28 @@ leaq 0+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 32(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, S2, in2_y); leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromq + leaq 96(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromq // p256_sub(R, S2, in1_y); movq 0+64(%rsp),%rax movq 8+64(%rsp),%r14 leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 128(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Hsqr, H); movq 0+96(%rsp),%rax movq 8+96(%rsp),%r14 leaq 0+96(%rsp),%rsi movq 16+96(%rsp),%r15 movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montq + leaq 192(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montq // p256_sqr_mont(Rsqr, R); movq 128(%rsp),%rax leaq 128(%rsp),%rbx @@ -3290,8 +3290,8 @@ leaq 0+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq + leaq 160(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montq // p256_mul_mont(Hcub, Hsqr, H); movq 320(%rsp),%rax leaq 320(%rsp),%rbx @@ -3300,14 +3300,14 @@ leaq 0+128(%rsp),%rsi movq 16+128(%rsp),%r11 movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - + leaq 0(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montq // p256_mul_mont(U2, in1_x, Hsqr); +// lea 0(%rsp), %rsi +// lea 128(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 192(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -3333,11 +3333,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(res_x, Rsqr, Hsqr); leaq 160(%rsp),%rbx leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_x, res_x, Hcub); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp @@ -3345,10 +3345,10 @@ movq 0+24(%rsp),%r10 leaq 64(%rsp),%rdi - call __ecp_nistz256_subq + call __ecp_nistz256_subq // p256_sub(H, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 352(%rsp),%rax @@ -3359,7 +3359,7 @@ movq 16+160(%rsp),%r11 movq 24+160(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(S2, Hcub, in1_y); movq 96(%rsp),%rax leaq 96(%rsp),%rbx @@ -3369,15 +3369,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq // p256_mul_mont(H, H, R); leaq 32(%rsp),%rbx leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq // p256_sub(res_y, H, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, ONE, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3388,7 +3388,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3401,7 +3401,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 224(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3412,7 +3412,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3425,7 +3425,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 256(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -3436,7 +3436,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -3575,7 +3575,7 @@ __ecp_nistz256_mul_by_2x: .cfi_startproc xorq %r11,%r11 - adcq %r12,%r12 + adcq %r12,%r12 // a0:a3+a0:a3 adcq %r13,%r13 movq %r12,%rax adcq %r8,%r8 @@ -3634,10 +3634,10 @@ .Lpoint_doublex_body: .Lpoint_double_shortcutx: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi.x + movq %rsi,%rbx // backup copy movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 + movq 32+0(%rsi),%r12 // load in_y in "5-4-0-1" order movq 32+8(%rsi),%r13 movq 32+16(%rsi),%r8 movq 32+24(%rsi),%r9 @@ -3652,7 +3652,7 @@ movq %r11,%xmm2 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_by_2x // p256_mul_by_2(S, in_y); movq 64+0(%rsi),%rdx movq 64+8(%rsi),%r14 @@ -3660,7 +3660,7 @@ movq 64+24(%rsi),%r8 leaq 64-128(%rsi),%rsi leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Zsqr, in_z); movq 0+0(%rsp),%rdx movq 8+0(%rsp),%r14 @@ -3668,9 +3668,9 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(S, S); - movq 32(%rbx),%rdx + movq 32(%rbx),%rdx // %rbx is still valid movq 64+0(%rbx),%r9 movq 64+8(%rbx),%r10 movq 64+16(%rbx),%r11 @@ -3678,24 +3678,24 @@ leaq 64-128(%rbx),%rsi leaq 32(%rbx),%rbx movq %xmm2,%rdi - call __ecp_nistz256_mul_montx - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2x // p256_mul_by_2(res_z, res_z); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox + call __ecp_nistz256_add_tox // p256_add(M, in_x, Zsqr); - movq 96+0(%rsp),%r12 + movq 96+0(%rsp),%r12 // "5-4-0-1" order movq 96+8(%rsp),%r13 leaq 64(%rsp),%rbx movq 96+16(%rsp),%r8 movq 96+24(%rsp),%r9 leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(Zsqr, in_x, Zsqr); movq 0+0(%rsp),%rdx movq 8+0(%rsp),%r14 @@ -3703,7 +3703,7 @@ movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 movq %xmm1,%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(res_y, S); xorq %r9,%r9 movq %r12,%rax addq $-1,%r12 @@ -3714,7 +3714,7 @@ movq %r15,%r8 adcq %rbp,%r15 adcq $0,%r9 - xorq %rsi,%rsi + xorq %rsi,%rsi // borrow %rsi testq $1,%rax cmovzq %rax,%r12 @@ -3723,7 +3723,7 @@ cmovzq %r8,%r15 cmovzq %rsi,%r9 - movq %r13,%rax + movq %r13,%rax // a0:a3>>1 shrq $1,%r12 shlq $63,%rax movq %r14,%r10 @@ -3750,14 +3750,14 @@ movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(M, M, Zsqr); leaq 128(%rsp),%rdi call __ecp_nistz256_mul_by_2x leaq 32(%rsp),%rbx leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox + call __ecp_nistz256_add_tox // p256_mul_by_3(M, M); movq 96(%rsp),%rdx leaq 96(%rsp),%rbx @@ -3767,10 +3767,10 @@ movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S, S, in_x); leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_by_2x // p256_mul_by_2(tmp0, S); movq 0+32(%rsp),%rdx movq 8+32(%rsp),%r14 @@ -3778,27 +3778,27 @@ movq 16+32(%rsp),%r15 movq 24+32(%rsp),%r8 movq %xmm0,%rdi - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx // p256_sqr_mont(res_x, M); leaq 128(%rsp),%rbx - movq %r14,%r8 + movq %r14,%r8 // harmonize sqr output and sub input movq %r15,%r9 movq %rsi,%r14 movq %rbp,%r15 - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_x, res_x, tmp0); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 + movq 0+24(%rsp),%r10 // "4-5-0-1" order leaq 0(%rsp),%rdi - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(S, S, res_x); movq 32(%rsp),%rdx leaq 32(%rsp),%rbx - movq %r12,%r14 + movq %r12,%r14 // harmonize sub output and mul input xorl %ecx,%ecx - movq %r12,0+0(%rsp) + movq %r12,0+0(%rsp) // have to save:-( movq %r13,%r10 movq %r13,0+8(%rsp) cmovzq %r8,%r11 @@ -3808,11 +3808,11 @@ movq %r9,0+24(%rsp) movq %r14,%r9 leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S, S, M); movq %xmm1,%rbx movq %xmm1,%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_y, S, res_y); leaq 160+56(%rsp),%rsi .cfi_def_cfa %rsi,8 @@ -3863,14 +3863,14 @@ .cfi_adjust_cfa_offset 32*18+8 .Lpoint_addx_body: - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi + movq %rsi,%rbx // reassign + movq %rdx,%rsi // reassign movdqa %xmm0,384(%rsp) movdqa %xmm1,384+16(%rsp) movdqa %xmm2,416(%rsp) @@ -3879,54 +3879,54 @@ movdqa %xmm5,448+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rsi),%xmm0 + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 por %xmm3,%xmm5 movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rdx + movq 64+0(%rsi),%rdx // load original in2_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 movdqa %xmm0,480(%rsp) pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 + movdqu 64(%rsi),%xmm0 // in2_z again movdqu 80(%rsi),%xmm1 movdqa %xmm2,512(%rsp) movdqa %xmm3,512+16(%rsp) por %xmm4,%xmm5 pxor %xmm4,%xmm4 por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi - leaq 64-128(%rsi),%rsi - movq %rdx,544+0(%rsp) + leaq 64-128(%rsi),%rsi // %rsi is still valid + movq %rdx,544+0(%rsp) // make in2_z copy movq %r14,544+8(%rsp) movq %r15,544+16(%rsp) movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 96(%rsp),%rdi // Z2^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Z2sqr, in2_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm1,%xmm4 por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 por %xmm3,%xmm4 pxor %xmm3,%xmm3 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rdx + pshufd $0,%xmm4,%xmm4 // in2infty + movq 64+0(%rbx),%rdx // load original in1_z movq 64+8(%rbx),%r14 movq 64+16(%rbx),%r15 movq 64+24(%rbx),%r8 movq %rbx,%xmm1 leaq 64-128(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Z1sqr, in1_z); movq 544(%rsp),%rdx leaq 544(%rsp),%rbx @@ -3935,8 +3935,8 @@ leaq -128+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 224(%rsp),%rdi // S1 = Z2^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S1, Z2sqr, in2_z); movq 448(%rsp),%rdx leaq 448(%rsp),%rbx @@ -3945,8 +3945,8 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 256(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, Z1sqr, in1_z); movq 416(%rsp),%rdx leaq 416(%rsp),%rbx @@ -3955,8 +3955,8 @@ leaq -128+224(%rsp),%rsi movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 224(%rsp),%rdi // S1 = Y1*Z2^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S1, S1, in1_y); movq 512(%rsp),%rdx leaq 512(%rsp),%rbx @@ -3965,18 +3965,18 @@ leaq -128+256(%rsp),%rsi movq 16+256(%rsp),%r11 movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 256(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, S2, in2_y); leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 64(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromx // p256_sub(R, S2, S1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero movdqa %xmm4,%xmm2 orq %r8,%r12 orq %r9,%r12 - por %xmm5,%xmm2 + por %xmm5,%xmm2 // in1infty || in2infty movq %r12,%xmm3 movq 384(%rsp),%rdx @@ -3986,8 +3986,8 @@ leaq -128+96(%rsp),%rsi movq 16+96(%rsp),%r11 movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 160(%rsp),%rdi // U1 = X1*Z2^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U1, in1_x, Z2sqr); movq 480(%rsp),%rdx leaq 480(%rsp),%rbx @@ -3996,34 +3996,34 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 192(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, in2_x, Z1sqr); leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 0(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromx // p256_sub(H, U2, U1); - orq %r13,%r12 + orq %r13,%r12 // see if result is zero orq %r8,%r12 - orq %r9,%r12 + orq %r9,%r12 // movq %xmm2,%r8 movq %xmm3,%r9 orq %r8,%r12 -.byte 0x3e - jnz .Ladd_proceedx +.byte 0x3e // predict taken + jnz .Ladd_proceedx // - - +// We now know A = B or A = -B and neither is infinity. Compare the +// y-coordinates via S1 and S2. testq %r9,%r9 - jz .Ladd_doublex + jz .Ladd_doublex // is_equal(S1, S2) - - - - - - movq %xmm0,%rdi +// A = -B, so the result is infinity. +// +// TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in +// which case we should eliminate this special-case and simplify the +// timing analysis. + movq %xmm0,%rdi // restore %rdi pxor %xmm0,%xmm0 movdqu %xmm0,0(%rdi) movdqu %xmm0,16(%rdi) @@ -4035,9 +4035,9 @@ .align 32 .Ladd_doublex: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp + movq %xmm1,%rsi // restore %rsi + movq %xmm0,%rdi // restore %rdi + addq $416,%rsp // difference in frame sizes .cfi_adjust_cfa_offset -416 jmp .Lpoint_double_shortcutx .cfi_adjust_cfa_offset 416 @@ -4049,8 +4049,8 @@ leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 96(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Rsqr, R); movq 448(%rsp),%rdx leaq 448(%rsp),%rbx @@ -4059,16 +4059,16 @@ leaq -128+0(%rsp),%rsi movq 16+0(%rsp),%r11 movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, H, in1_z); movq 0+0(%rsp),%rdx movq 8+0(%rsp),%r14 leaq -128+0(%rsp),%rsi movq 16+0(%rsp),%r15 movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 32(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Hsqr, H); movq 544(%rsp),%rdx leaq 544(%rsp),%rbx @@ -4077,8 +4077,8 @@ leaq -128+352(%rsp),%rsi movq 16+352(%rsp),%r11 movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 352(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, res_z, in2_z); movq 0(%rsp),%rdx leaq 0(%rsp),%rbx @@ -4087,8 +4087,8 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 128(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(Hcub, Hsqr, H); movq 160(%rsp),%rdx leaq 160(%rsp),%rbx @@ -4097,14 +4097,14 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - + leaq 192(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, U1, Hsqr); +// lea 192(%rsp), %rsi +// lea 32(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 96(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -4130,11 +4130,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(res_x, Rsqr, Hsqr); leaq 128(%rsp),%rbx leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_x, res_x, Hcub); movq 192+0(%rsp),%rax movq 192+8(%rsp),%rbp @@ -4142,10 +4142,10 @@ movq 192+24(%rsp),%r10 leaq 320(%rsp),%rdi - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(res_y, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 128(%rsp),%rdx @@ -4156,7 +4156,7 @@ movq 16+224(%rsp),%r11 movq 24+224(%rsp),%r12 leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, S1, Hcub); movq 320(%rsp),%rdx leaq 320(%rsp),%rbx @@ -4166,15 +4166,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(res_y, R, res_y); leaq 256(%rsp),%rbx leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_y, res_y, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, in2_z, in1infty); movdqa %xmm5,%xmm1 pandn 352(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4185,7 +4185,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4198,7 +4198,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4209,7 +4209,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4222,7 +4222,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 320(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4233,7 +4233,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4296,14 +4296,14 @@ .cfi_adjust_cfa_offset 32*15+8 .Ladd_affinex_body: - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx + movdqu 0(%rsi),%xmm0 // copy *(P256_POINT *)%rsi + movq %rdx,%rbx // reassign movdqu 16(%rsi),%xmm1 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm3 movdqu 64(%rsi),%xmm4 movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rdx + movq 64+0(%rsi),%rdx // load original in1_z movq 64+8(%rsi),%r14 movq 64+16(%rsi),%r15 movq 64+24(%rsi),%r8 @@ -4315,7 +4315,7 @@ movdqa %xmm5,384+16(%rsp) por %xmm4,%xmm5 - movdqu 0(%rbx),%xmm0 + movdqu 0(%rbx),%xmm0 // copy *(P256_POINT_AFFINE *)%rbx pshufd $0xb1,%xmm5,%xmm3 movdqu 16(%rbx),%xmm1 movdqu 32(%rbx),%xmm2 @@ -4325,7 +4325,7 @@ pshufd $0x1e,%xmm5,%xmm4 movdqa %xmm1,416+16(%rsp) por %xmm0,%xmm1 - movq %rdi,%xmm0 + movq %rdi,%xmm0 // save %rdi movdqa %xmm2,448(%rsp) movdqa %xmm3,448+16(%rsp) por %xmm2,%xmm3 @@ -4333,33 +4333,33 @@ pxor %xmm4,%xmm4 por %xmm1,%xmm3 - leaq 64-128(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 64-128(%rsi),%rsi // %rsi is still valid + leaq 32(%rsp),%rdi // Z1^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Z1sqr, in1_z); pcmpeqd %xmm4,%xmm5 pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rdx - - movq %r12,%r9 + movq 0(%rbx),%rdx // %rbx is still valid +// lea 0x00(%rbx), %rbx + movq %r12,%r9 // harmonize sqr output and mul input por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // in1infty pshufd $0x1e,%xmm4,%xmm3 movq %r13,%r10 por %xmm3,%xmm4 pxor %xmm3,%xmm3 movq %r14,%r11 pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 + pshufd $0,%xmm4,%xmm4 // in2infty leaq 32-128(%rsp),%rsi movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 0(%rsp),%rdi // U2 = X2*Z1^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, Z1sqr, in2_x); leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 64(%rsp),%rdi // H = U2 - U1 + call __ecp_nistz256_sub_fromx // p256_sub(H, U2, in1_x); movq 384(%rsp),%rdx leaq 384(%rsp),%rbx @@ -4368,8 +4368,8 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 32(%rsp),%rdi // S2 = Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, Z1sqr, in1_z); movq 384(%rsp),%rdx leaq 384(%rsp),%rbx @@ -4378,8 +4378,8 @@ leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 288(%rsp),%rdi // Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx // p256_mul_mont(res_z, H, in1_z); movq 448(%rsp),%rdx leaq 448(%rsp),%rbx @@ -4388,28 +4388,28 @@ leaq -128+32(%rsp),%rsi movq 16+32(%rsp),%r11 movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 32(%rsp),%rdi // S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, S2, in2_y); leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromx + leaq 96(%rsp),%rdi // R = S2 - S1 + call __ecp_nistz256_sub_fromx // p256_sub(R, S2, in1_y); movq 0+64(%rsp),%rdx movq 8+64(%rsp),%r14 leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r15 movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 128(%rsp),%rdi // H^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Hsqr, H); movq 0+96(%rsp),%rdx movq 8+96(%rsp),%r14 leaq -128+96(%rsp),%rsi movq 16+96(%rsp),%r15 movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montx + leaq 192(%rsp),%rdi // R^2 + call __ecp_nistz256_sqr_montx // p256_sqr_mont(Rsqr, R); movq 128(%rsp),%rdx leaq 128(%rsp),%rbx @@ -4418,8 +4418,8 @@ leaq -128+64(%rsp),%rsi movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx + leaq 160(%rsp),%rdi // H^3 + call __ecp_nistz256_mul_montx // p256_mul_mont(Hcub, Hsqr, H); movq 320(%rsp),%rdx leaq 320(%rsp),%rbx @@ -4428,14 +4428,14 @@ leaq -128+128(%rsp),%rsi movq 16+128(%rsp),%r11 movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - + leaq 0(%rsp),%rdi // U1*H^2 + call __ecp_nistz256_mul_montx // p256_mul_mont(U2, in1_x, Hsqr); +// lea 0(%rsp), %rsi +// lea 128(%rsp), %rdi # 2*U1*H^2 +// call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xorq %r11,%r11 - addq %r12,%r12 + addq %r12,%r12 // a0:a3+a0:a3 leaq 192(%rsp),%rsi adcq %r13,%r13 movq %r12,%rax @@ -4461,11 +4461,11 @@ cmovcq %r10,%r9 movq 24(%rsi),%r10 - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(res_x, Rsqr, Hsqr); leaq 160(%rsp),%rbx leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_x, res_x, Hcub); movq 0+0(%rsp),%rax movq 0+8(%rsp),%rbp @@ -4473,10 +4473,10 @@ movq 0+24(%rsp),%r10 leaq 64(%rsp),%rdi - call __ecp_nistz256_subx + call __ecp_nistz256_subx // p256_sub(H, U2, res_x); - movq %r12,0(%rdi) - movq %r13,8(%rdi) + movq %r12,0(%rdi) // save the result, as + movq %r13,8(%rdi) // __ecp_nistz256_sub doesn't movq %r8,16(%rdi) movq %r9,24(%rdi) movq 352(%rsp),%rdx @@ -4487,7 +4487,7 @@ movq 16+160(%rsp),%r11 movq 24+160(%rsp),%r12 leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(S2, Hcub, in1_y); movq 96(%rsp),%rdx leaq 96(%rsp),%rbx @@ -4497,15 +4497,15 @@ movq 16+64(%rsp),%r11 movq 24+64(%rsp),%r12 leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx // p256_mul_mont(H, H, R); leaq 32(%rsp),%rbx leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx // p256_sub(res_y, H, S2); - movq %xmm0,%rdi + movq %xmm0,%rdi // restore %rdi - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_z, ONE, in1infty); movdqa %xmm5,%xmm1 pandn 288(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4516,7 +4516,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_z, in1_z, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4529,7 +4529,7 @@ movdqu %xmm2,64(%rdi) movdqu %xmm3,80(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_x, in2_x, in1infty); movdqa %xmm5,%xmm1 pandn 224(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4540,7 +4540,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_x, in1_x, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2 @@ -4553,7 +4553,7 @@ movdqu %xmm2,0(%rdi) movdqu %xmm3,16(%rdi) - movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm0 // copy_conditional(res_y, in2_y, in1infty); movdqa %xmm5,%xmm1 pandn 256(%rsp),%xmm0 movdqa %xmm5,%xmm2 @@ -4564,7 +4564,7 @@ por %xmm0,%xmm2 por %xmm1,%xmm3 - movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm0 // copy_conditional(res_y, in1_y, in2infty); movdqa %xmm4,%xmm1 pandn %xmm2,%xmm0 movdqa %xmm4,%xmm2
diff --git a/gen/bcm/p256-x86_64-asm-win.asm b/gen/bcm/p256-x86_64-asm-win.asm index 232800b..9976ab3 100644 --- a/gen/bcm/p256-x86_64-asm-win.asm +++ b/gen/bcm/p256-x86_64-asm-win.asm
@@ -14,7 +14,7 @@ section .text code align=64 - +; The polynomial section .rdata rdata align=8 ALIGN 64 p256_constants: @@ -30,7 +30,7 @@ $L$ONE_mont: DQ 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe - +; Constants for computations modulo ord(p256) $L$ord: DQ 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 $L$ordK: @@ -38,8 +38,8 @@ section .text - - +; ############################################################################### +; void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]); global ecp_nistz256_neg ALIGN 32 @@ -104,11 +104,11 @@ ret $L$SEH_end_ecp_nistz256_neg: - - - - - +; ############################################################################### +; void ecp_nistz256_ord_mul_mont( +; uint64_t res[4], +; uint64_t a[4], +; uint64_t b[4]); global ecp_nistz256_ord_mul_mont_nohw @@ -144,7 +144,7 @@ lea r14,[$L$ord] mov r15,QWORD[$L$ordK] - +; ################################ * b[0] mov rcx,rax mul QWORD[rsi] mov r8,rax @@ -172,16 +172,16 @@ adc rdx,0 mov r12,rdx - +; ################################ First reduction step mul QWORD[r14] mov rbp,r8 - add r13,rax + add r13,rax ; guaranteed to be zero mov rax,r8 adc rdx,0 mov rcx,rdx sub r10,r8 - sbb r8,0 + sbb r8,0 ; can't borrow mul QWORD[8+r14] add r9,rcx @@ -190,19 +190,19 @@ mov rax,rbp adc r10,rdx mov rdx,rbp - adc r8,0 + adc r8,0 ; can't overflow shl rax,32 shr rdx,32 sub r11,rax mov rax,QWORD[8+rbx] - sbb rbp,rdx + sbb rbp,rdx ; can't borrow add r11,r8 adc r12,rbp adc r13,0 - +; ################################ * b[1] mov rcx,rax mul QWORD[rsi] add r9,rax @@ -238,15 +238,15 @@ adc r13,rdx adc r8,0 - +; ################################ Second reduction step mul QWORD[r14] mov rbp,r9 - add rcx,rax + add rcx,rax ; guaranteed to be zero mov rax,r9 adc rcx,rdx sub r11,r9 - sbb r9,0 + sbb r9,0 ; can't borrow mul QWORD[8+r14] add r10,rcx @@ -255,19 +255,19 @@ mov rax,rbp adc r11,rdx mov rdx,rbp - adc r9,0 + adc r9,0 ; can't overflow shl rax,32 shr rdx,32 sub r12,rax mov rax,QWORD[16+rbx] - sbb rbp,rdx + sbb rbp,rdx ; can't borrow add r12,r9 adc r13,rbp adc r8,0 - +; ################################# * b[2] mov rcx,rax mul QWORD[rsi] add r10,rax @@ -303,15 +303,15 @@ adc r8,rdx adc r9,0 - +; ################################ Third reduction step mul QWORD[r14] mov rbp,r10 - add rcx,rax + add rcx,rax ; guaranteed to be zero mov rax,r10 adc rcx,rdx sub r12,r10 - sbb r10,0 + sbb r10,0 ; can't borrow mul QWORD[8+r14] add r11,rcx @@ -320,19 +320,19 @@ mov rax,rbp adc r12,rdx mov rdx,rbp - adc r10,0 + adc r10,0 ; can't overflow shl rax,32 shr rdx,32 sub r13,rax mov rax,QWORD[24+rbx] - sbb rbp,rdx + sbb rbp,rdx ; can't borrow add r13,r10 adc r8,rbp adc r9,0 - +; ################################ * b[3] mov rcx,rax mul QWORD[rsi] add r11,rax @@ -368,15 +368,15 @@ adc r9,rdx adc r10,0 - +; ################################ Last reduction step mul QWORD[r14] mov rbp,r11 - add rcx,rax + add rcx,rax ; guaranteed to be zero mov rax,r11 adc rcx,rdx sub r13,r11 - sbb r11,0 + sbb r11,0 ; can't borrow mul QWORD[8+r14] add r12,rcx @@ -385,18 +385,18 @@ mov rax,rbp adc r13,rdx mov rdx,rbp - adc r11,0 + adc r11,0 ; can't overflow shl rax,32 shr rdx,32 sub r8,rax - sbb rbp,rdx + sbb rbp,rdx ; can't borrow add r8,r11 adc r9,rbp adc r10,0 - +; ################################ Subtract ord mov rsi,r12 sub r12,QWORD[r14] mov r11,r13 @@ -438,11 +438,11 @@ $L$SEH_end_ecp_nistz256_ord_mul_mont_nohw: - - - - - +; ############################################################################### +; void ecp_nistz256_ord_sqr_mont( +; uint64_t res[4], +; uint64_t a[4], +; uint64_t rep); global ecp_nistz256_ord_sqr_mont_nohw @@ -477,56 +477,56 @@ mov rax,QWORD[8+rsi] mov r14,QWORD[16+rsi] mov r15,QWORD[24+rsi] - lea rsi,[$L$ord] + lea rsi,[$L$ord] ; pointer to modulus mov rbx,rdx jmp NEAR $L$oop_ord_sqr ALIGN 32 $L$oop_ord_sqr: - - mov rbp,rax - mul r8 +; ################################ a[1:] * a[0] + mov rbp,rax ; put aside a[1] + mul r8 ; a[1] * a[0] mov r9,rax - movq xmm1,rbp + movq xmm1,rbp ; offload a[1] mov rax,r14 mov r10,rdx - mul r8 + mul r8 ; a[2] * a[0] add r10,rax mov rax,r15 - movq xmm2,r14 + movq xmm2,r14 ; offload a[2] adc rdx,0 mov r11,rdx - mul r8 + mul r8 ; a[3] * a[0] add r11,rax mov rax,r15 - movq xmm3,r15 + movq xmm3,r15 ; offload a[3] adc rdx,0 mov r12,rdx - - mul r14 +; ################################ a[3] * a[2] + mul r14 ; a[3] * a[2] mov r13,rax mov rax,r14 mov r14,rdx - - mul rbp +; ################################ a[2:] * a[1] + mul rbp ; a[2] * a[1] add r11,rax mov rax,r15 adc rdx,0 mov r15,rdx - mul rbp + mul rbp ; a[3] * a[1] add r12,rax adc rdx,0 add r12,r15 adc r13,rdx - adc r14,0 + adc r14,0 ; can't overflow - +; ################################ *2 xor r15,r15 mov rax,r8 add r9,r9 @@ -537,20 +537,20 @@ adc r14,r14 adc r15,0 - - mul rax +; ################################ Missing products + mul rax ; a[0] * a[0] mov r8,rax movq rax,xmm1 mov rbp,rdx - mul rax + mul rax ; a[1] * a[1] add r9,rbp adc r10,rax movq rax,xmm2 adc rdx,0 mov rbp,rdx - mul rax + mul rax ; a[2] * a[2] add r11,rbp adc r12,rax movq rax,xmm3 @@ -558,23 +558,23 @@ mov rbp,rdx mov rcx,r8 - imul r8,QWORD[32+rsi] + imul r8,QWORD[32+rsi] ; *= .LordK - mul rax + mul rax ; a[3] * a[3] add r13,rbp adc r14,rax - mov rax,QWORD[rsi] - adc r15,rdx + mov rax,QWORD[rsi] ; modulus[0] + adc r15,rdx ; can't overflow - +; ################################ First reduction step mul r8 mov rbp,r8 - add rcx,rax - mov rax,QWORD[8+rsi] + add rcx,rax ; guaranteed to be zero + mov rax,QWORD[8+rsi] ; modulus[1] adc rcx,rdx sub r10,r8 - sbb rbp,0 + sbb rbp,0 ; can't borrow mul r8 add r9,rcx @@ -583,29 +583,29 @@ mov rax,r8 adc r10,rdx mov rdx,r8 - adc rbp,0 + adc rbp,0 ; can't overflow mov rcx,r9 - imul r9,QWORD[32+rsi] + imul r9,QWORD[32+rsi] ; *= .LordK shl rax,32 shr rdx,32 sub r11,rax mov rax,QWORD[rsi] - sbb r8,rdx + sbb r8,rdx ; can't borrow add r11,rbp - adc r8,0 + adc r8,0 ; can't overflow - +; ################################ Second reduction step mul r9 mov rbp,r9 - add rcx,rax + add rcx,rax ; guaranteed to be zero mov rax,QWORD[8+rsi] adc rcx,rdx sub r11,r9 - sbb rbp,0 + sbb rbp,0 ; can't borrow mul r9 add r10,rcx @@ -614,29 +614,29 @@ mov rax,r9 adc r11,rdx mov rdx,r9 - adc rbp,0 + adc rbp,0 ; can't overflow mov rcx,r10 - imul r10,QWORD[32+rsi] + imul r10,QWORD[32+rsi] ; *= .LordK shl rax,32 shr rdx,32 sub r8,rax mov rax,QWORD[rsi] - sbb r9,rdx + sbb r9,rdx ; can't borrow add r8,rbp - adc r9,0 + adc r9,0 ; can't overflow - +; ################################ Third reduction step mul r10 mov rbp,r10 - add rcx,rax + add rcx,rax ; guaranteed to be zero mov rax,QWORD[8+rsi] adc rcx,rdx sub r8,r10 - sbb rbp,0 + sbb rbp,0 ; can't borrow mul r10 add r11,rcx @@ -645,29 +645,29 @@ mov rax,r10 adc r8,rdx mov rdx,r10 - adc rbp,0 + adc rbp,0 ; can't overflow mov rcx,r11 - imul r11,QWORD[32+rsi] + imul r11,QWORD[32+rsi] ; *= .LordK shl rax,32 shr rdx,32 sub r9,rax mov rax,QWORD[rsi] - sbb r10,rdx + sbb r10,rdx ; can't borrow add r9,rbp - adc r10,0 + adc r10,0 ; can't overflow - +; ################################ Last reduction step mul r11 mov rbp,r11 - add rcx,rax + add rcx,rax ; guaranteed to be zero mov rax,QWORD[8+rsi] adc rcx,rdx sub r9,r11 - sbb rbp,0 + sbb rbp,0 ; can't borrow mul r11 add r8,rcx @@ -676,17 +676,17 @@ mov rax,r11 adc r9,rdx mov rdx,r11 - adc rbp,0 + adc rbp,0 ; can't overflow shl rax,32 shr rdx,32 sub r10,rax - sbb r11,rdx + sbb r11,rdx ; can't borrow add r10,rbp - adc r11,0 + adc r11,0 ; can't overflow - +; ################################ Add bits [511:256] of the sqr result xor rdx,rdx add r8,r12 adc r9,r13 @@ -696,7 +696,7 @@ mov rax,r9 adc rdx,0 - +; ################################ Compare to modulus sub r8,QWORD[rsi] mov r14,r10 sbb r9,QWORD[8+rsi] @@ -741,7 +741,7 @@ ret $L$SEH_end_ecp_nistz256_ord_sqr_mont_nohw: - +; ############################################################################### global ecp_nistz256_ord_mul_mont_adx ALIGN 32 @@ -778,11 +778,11 @@ mov r10,QWORD[8+rsi] mov r11,QWORD[16+rsi] mov r12,QWORD[24+rsi] - lea rsi,[((-128))+rsi] + lea rsi,[((-128))+rsi] ; control u-op density lea r14,[(($L$ord-128))] mov r15,QWORD[$L$ordK] - +; ################################ Multiply by b[0] mulx r9,r8,r9 mulx r10,rcx,r10 mulx r11,rbp,r11 @@ -794,10 +794,10 @@ adc r11,rcx adc r12,0 - - xor r13,r13 +; ################################ reduction + xor r13,r13 ; %r13=0, cf=0, of=0 mulx rbp,rcx,QWORD[((0+128))+r14] - adcx r8,rcx + adcx r8,rcx ; guaranteed to be zero adox r9,rbp mulx rbp,rcx,QWORD[((8+128))+r14] @@ -814,9 +814,9 @@ adox r12,rbp adcx r12,r8 adox r13,r8 - adc r13,0 + adc r13,0 ; cf=0, of=0 - +; ################################ Multiply by b[1] mulx rbp,rcx,QWORD[((0+128))+rsi] adcx r9,rcx adox r10,rbp @@ -837,11 +837,11 @@ adcx r13,r8 adox r8,r8 - adc r8,0 + adc r8,0 ; cf=0, of=0 - +; ################################ reduction mulx rbp,rcx,QWORD[((0+128))+r14] - adcx r9,rcx + adcx r9,rcx ; guaranteed to be zero adox r10,rbp mulx rbp,rcx,QWORD[((8+128))+r14] @@ -858,9 +858,9 @@ adox r13,rbp adcx r13,r9 adox r8,r9 - adc r8,0 + adc r8,0 ; cf=0, of=0 - +; ################################ Multiply by b[2] mulx rbp,rcx,QWORD[((0+128))+rsi] adcx r10,rcx adox r11,rbp @@ -881,11 +881,11 @@ adcx r8,r9 adox r9,r9 - adc r9,0 + adc r9,0 ; cf=0, of=0 - +; ################################ reduction mulx rbp,rcx,QWORD[((0+128))+r14] - adcx r10,rcx + adcx r10,rcx ; guaranteed to be zero adox r11,rbp mulx rbp,rcx,QWORD[((8+128))+r14] @@ -902,9 +902,9 @@ adox r8,rbp adcx r8,r10 adox r9,r10 - adc r9,0 + adc r9,0 ; cf=0, of=0 - +; ################################ Multiply by b[3] mulx rbp,rcx,QWORD[((0+128))+rsi] adcx r11,rcx adox r12,rbp @@ -925,11 +925,11 @@ adcx r9,r10 adox r10,r10 - adc r10,0 + adc r10,0 ; cf=0, of=0 - +; ################################ reduction mulx rbp,rcx,QWORD[((0+128))+r14] - adcx r11,rcx + adcx r11,rcx ; guaranteed to be zero adox r12,rbp mulx rbp,rcx,QWORD[((8+128))+r14] @@ -950,8 +950,8 @@ adox r10,r11 adc r10,0 - - +; ################################ +; Branch-less conditional subtraction of P mov rcx,r8 sub r12,QWORD[r14] sbb r13,QWORD[8+r14] @@ -1031,38 +1031,38 @@ ALIGN 32 $L$oop_ord_sqrx: - mulx r10,r9,r14 - mulx r11,rcx,r15 - mov rax,rdx - movq xmm1,r14 - mulx r12,rbp,r8 + mulx r10,r9,r14 ; a[0]*a[1] + mulx r11,rcx,r15 ; a[0]*a[2] + mov rax,rdx ; offload a[0] + movq xmm1,r14 ; offload a[1] + mulx r12,rbp,r8 ; a[0]*a[3] mov rdx,r14 add r10,rcx - movq xmm2,r15 + movq xmm2,r15 ; offload a[2] adc r11,rbp adc r12,0 - xor r13,r13 - - mulx rbp,rcx,r15 + xor r13,r13 ; %r13=0,cf=0,of=0 +; ################################ + mulx rbp,rcx,r15 ; a[1]*a[2] adcx r11,rcx adox r12,rbp - mulx rbp,rcx,r8 + mulx rbp,rcx,r8 ; a[1]*a[3] mov rdx,r15 adcx r12,rcx adox r13,rbp adc r13,0 - - mulx r14,rcx,r8 +; ################################ + mulx r14,rcx,r8 ; a[2]*a[3] mov rdx,rax - movq xmm3,r8 - xor r15,r15 - adcx r9,r9 + movq xmm3,r8 ; offload a[3] + xor r15,r15 ; %r15=0,cf=0,of=0 + adcx r9,r9 ; acc1:6<<1 adox r13,rcx adcx r10,r10 - adox r14,r15 + adox r14,r15 ; of=0 - +; ################################ a[i]*a[i] mulx rbp,r8,rdx movq rdx,xmm1 adcx r11,r11 @@ -1084,13 +1084,13 @@ adox r14,rcx adox r15,rax - +; ################################ reduction mov rdx,r8 mulx rcx,rdx,QWORD[32+rsi] - xor rax,rax + xor rax,rax ; cf=0, of=0 mulx rbp,rcx,QWORD[rsi] - adcx r8,rcx + adcx r8,rcx ; guaranteed to be zero adox r9,rbp mulx rbp,rcx,QWORD[8+rsi] adcx r9,rcx @@ -1100,15 +1100,15 @@ adox r11,rbp mulx rbp,rcx,QWORD[24+rsi] adcx r11,rcx - adox r8,rbp - adcx r8,rax + adox r8,rbp ; of=0 + adcx r8,rax ; cf=0 - +; ################################ mov rdx,r9 mulx rcx,rdx,QWORD[32+rsi] mulx rbp,rcx,QWORD[rsi] - adox r9,rcx + adox r9,rcx ; guaranteed to be zero adcx r10,rbp mulx rbp,rcx,QWORD[8+rsi] adox r10,rcx @@ -1118,15 +1118,15 @@ adcx r8,rbp mulx rbp,rcx,QWORD[24+rsi] adox r8,rcx - adcx r9,rbp - adox r9,rax + adcx r9,rbp ; cf=0 + adox r9,rax ; of=0 - +; ################################ mov rdx,r10 mulx rcx,rdx,QWORD[32+rsi] mulx rbp,rcx,QWORD[rsi] - adcx r10,rcx + adcx r10,rcx ; guaranteed to be zero adox r11,rbp mulx rbp,rcx,QWORD[8+rsi] adcx r11,rcx @@ -1136,15 +1136,15 @@ adox r9,rbp mulx rbp,rcx,QWORD[24+rsi] adcx r9,rcx - adox r10,rbp - adcx r10,rax + adox r10,rbp ; of=0 + adcx r10,rax ; cf=0 - +; ################################ mov rdx,r11 mulx rcx,rdx,QWORD[32+rsi] mulx rbp,rcx,QWORD[rsi] - adox r11,rcx + adox r11,rcx ; guaranteed to be zero adcx r8,rbp mulx rbp,rcx,QWORD[8+rsi] adox r8,rcx @@ -1157,8 +1157,8 @@ adcx r11,rbp adox r11,rax - - add r12,r8 +; ################################ accumulate upper half + add r12,r8 ; add %r12, %r8 adc r9,r13 mov rdx,r12 adc r10,r14 @@ -1166,7 +1166,7 @@ mov r14,r9 adc rax,0 - +; ################################ compare to modulus sub r12,QWORD[rsi] mov r15,r10 sbb r9,QWORD[8+rsi] @@ -1211,11 +1211,11 @@ ret $L$SEH_end_ecp_nistz256_ord_sqr_mont_adx: - - - - - +; ############################################################################### +; void ecp_nistz256_mul_mont( +; uint64_t res[4], +; uint64_t a[4], +; uint64_t b[4]); global ecp_nistz256_mul_mont_nohw @@ -1279,8 +1279,8 @@ ALIGN 32 __ecp_nistz256_mul_montq: - - +; ####################################################################### +; Multiply a by b[0] mov rbp,rax mul r9 mov r14,QWORD[(($L$poly+8))] @@ -1308,20 +1308,20 @@ xor r13,r13 mov r12,rdx - - - - - - - - +; ####################################################################### +; First reduction step +; Basically now we want to multiply acc[0] by p256, +; and add the result to the acc. +; Due to the special form of p256 we do some optimizations +; +; acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0] +; then we add acc[0] and get acc[0] x 2^96 mov rbp,r8 shl r8,32 mul r15 shr rbp,32 - add r9,r8 + add r9,r8 ; +=acc[0]<<96 adc r10,rbp adc r11,rax mov rax,QWORD[8+rbx] @@ -1329,8 +1329,8 @@ adc r13,0 xor r8,r8 - - +; ####################################################################### +; Multiply by b[1] mov rbp,rax mul QWORD[rsi] add r9,rax @@ -1362,8 +1362,8 @@ adc r13,rdx adc r8,0 - - +; ####################################################################### +; Second reduction step mov rbp,r9 shl r9,32 mul r15 @@ -1376,8 +1376,8 @@ adc r8,0 xor r9,r9 - - +; ####################################################################### +; Multiply by b[2] mov rbp,rax mul QWORD[rsi] add r10,rax @@ -1409,8 +1409,8 @@ adc r8,rdx adc r9,0 - - +; ####################################################################### +; Third reduction step mov rbp,r10 shl r10,32 mul r15 @@ -1423,8 +1423,8 @@ adc r9,0 xor r10,r10 - - +; ####################################################################### +; Multiply by b[3] mov rbp,rax mul QWORD[rsi] add r11,rax @@ -1456,8 +1456,8 @@ adc r9,rdx adc r10,0 - - +; ####################################################################### +; Final reduction step mov rbp,r11 shl r11,32 mul r15 @@ -1470,14 +1470,14 @@ mov rbp,r13 adc r10,0 - - - sub r12,-1 +; ####################################################################### +; Branch-less conditional subtraction of P + sub r12,-1 ; .Lpoly[0] mov rbx,r8 - sbb r13,r14 - sbb r8,0 + sbb r13,r14 ; .Lpoly[1] + sbb r8,0 ; .Lpoly[2] mov rdx,r9 - sbb r9,r15 + sbb r9,r15 ; .Lpoly[3] sbb r10,0 cmovc r12,rcx @@ -1493,13 +1493,13 @@ +; ############################################################################### +; void ecp_nistz256_sqr_mont( +; uint64_t res[4], +; uint64_t a[4]); - - - - - - +; we optimize the square according to S.Gueron and V.Krasnov, +; "Speeding up Big-Number Squaring" global ecp_nistz256_sqr_mont_nohw ALIGN 32 @@ -1560,31 +1560,31 @@ __ecp_nistz256_sqr_montq: mov r13,rax - mul r14 + mul r14 ; a[1]*a[0] mov r9,rax mov rax,r15 mov r10,rdx - mul r13 + mul r13 ; a[0]*a[2] add r10,rax mov rax,r8 adc rdx,0 mov r11,rdx - mul r13 + mul r13 ; a[0]*a[3] add r11,rax mov rax,r15 adc rdx,0 mov r12,rdx - - mul r14 +; ################################ + mul r14 ; a[1]*a[2] add r11,rax mov rax,r8 adc rdx,0 mov rbp,rdx - mul r14 + mul r14 ; a[1]*a[3] add r12,rax mov rax,r8 adc rdx,0 @@ -1592,15 +1592,15 @@ mov r13,rdx adc r13,0 - - mul r15 +; ################################ + mul r15 ; a[2]*a[3] xor r15,r15 add r13,rax mov rax,QWORD[rsi] mov r14,rdx adc r14,0 - add r9,r9 + add r9,r9 ; acc1:6<<1 adc r10,r10 adc r11,r11 adc r12,r12 @@ -1636,21 +1636,21 @@ mov rsi,QWORD[(($L$poly+8))] mov rbp,QWORD[(($L$poly+24))] - - - +; ######################################### +; Now the reduction +; First iteration mov rcx,r8 shl r8,32 mul rbp shr rcx,32 - add r9,r8 + add r9,r8 ; +=acc[0]<<96 adc r10,rcx adc r11,rax mov rax,r9 adc rdx,0 - - +; ######################################### +; Second iteration mov rcx,r9 shl r9,32 mov r8,rdx @@ -1662,8 +1662,8 @@ mov rax,r10 adc rdx,0 - - +; ######################################### +; Third iteration mov rcx,r10 shl r10,32 mov r9,rdx @@ -1675,8 +1675,8 @@ mov rax,r11 adc rdx,0 - - +; ########################################## +; Last iteration mov rcx,r11 shl r11,32 mov r10,rdx @@ -1688,8 +1688,8 @@ adc rdx,0 xor r11,r11 - - +; ########################################### +; Add the rest of the acc add r12,r8 adc r13,r9 mov r8,r12 @@ -1698,12 +1698,12 @@ mov r9,r13 adc r11,0 - sub r12,-1 + sub r12,-1 ; .Lpoly[0] mov r10,r14 - sbb r13,rsi - sbb r14,0 + sbb r13,rsi ; .Lpoly[1] + sbb r14,0 ; .Lpoly[2] mov rcx,r15 - sbb r15,rbp + sbb r15,rbp ; .Lpoly[3] sbb r11,0 cmovc r12,r8 @@ -1752,7 +1752,7 @@ mov r10,QWORD[8+rsi] mov r11,QWORD[16+rsi] mov r12,QWORD[24+rsi] - lea rsi,[((-128))+rsi] + lea rsi,[((-128))+rsi] ; control u-op density call __ecp_nistz256_mul_montx @@ -1781,12 +1781,12 @@ ALIGN 32 __ecp_nistz256_mul_montx: - - +; ####################################################################### +; Multiply by b[0] mulx r9,r8,r9 mulx r10,rcx,r10 mov r14,32 - xor r13,r13 + xor r13,r13 ; cf=0 mulx r11,rbp,r11 mov r15,QWORD[(($L$poly+24))] adc r9,rcx @@ -1798,8 +1798,8 @@ shrx rcx,r8,r14 adc r12,0 - - +; ####################################################################### +; First reduction step add r9,rbp adc r10,rcx @@ -1808,10 +1808,10 @@ adc r11,rcx adc r12,rbp adc r13,0 - xor r8,r8 + xor r8,r8 ; %r8=0,cf=0,of=0 - - +; ####################################################################### +; Multiply by b[1] mulx rbp,rcx,QWORD[((0+128))+rsi] adcx r9,rcx adox r10,rbp @@ -1835,8 +1835,8 @@ adox r8,r8 adc r8,0 - - +; ####################################################################### +; Second reduction step add r10,rcx adc r11,rbp @@ -1845,10 +1845,10 @@ adc r12,rcx adc r13,rbp adc r8,0 - xor r9,r9 + xor r9,r9 ; %r9=0,cf=0,of=0 - - +; ####################################################################### +; Multiply by b[2] mulx rbp,rcx,QWORD[((0+128))+rsi] adcx r10,rcx adox r11,rbp @@ -1872,8 +1872,8 @@ adox r9,r9 adc r9,0 - - +; ####################################################################### +; Third reduction step add r11,rcx adc r12,rbp @@ -1882,10 +1882,10 @@ adc r13,rcx adc r8,rbp adc r9,0 - xor r10,r10 + xor r10,r10 ; %r10=0,cf=0,of=0 - - +; ####################################################################### +; Multiply by b[3] mulx rbp,rcx,QWORD[((0+128))+rsi] adcx r11,rcx adox r12,rbp @@ -1909,8 +1909,8 @@ adox r10,r10 adc r10,0 - - +; ####################################################################### +; Fourth reduction step add r12,rcx adc r13,rbp @@ -1922,15 +1922,15 @@ adc r9,rbp adc r10,0 - - +; ####################################################################### +; Branch-less conditional subtraction of P xor eax,eax mov rcx,r8 - sbb r12,-1 - sbb r13,r14 - sbb r8,0 + sbb r12,-1 ; .Lpoly[0] + sbb r13,r14 ; .Lpoly[1] + sbb r8,0 ; .Lpoly[2] mov rbp,r9 - sbb r9,r15 + sbb r9,r15 ; .Lpoly[3] sbb r10,0 cmovc r12,rbx @@ -1977,7 +1977,7 @@ mov r14,QWORD[8+rsi] mov r15,QWORD[16+rsi] mov r8,QWORD[24+rsi] - lea rsi,[((-128))+rsi] + lea rsi,[((-128))+rsi] ; control u-op density call __ecp_nistz256_sqr_montx @@ -2006,35 +2006,35 @@ ALIGN 32 __ecp_nistz256_sqr_montx: - mulx r10,r9,r14 - mulx r11,rcx,r15 + mulx r10,r9,r14 ; a[0]*a[1] + mulx r11,rcx,r15 ; a[0]*a[2] xor eax,eax adc r10,rcx - mulx r12,rbp,r8 + mulx r12,rbp,r8 ; a[0]*a[3] mov rdx,r14 adc r11,rbp adc r12,0 - xor r13,r13 + xor r13,r13 ; %r13=0,cf=0,of=0 - - mulx rbp,rcx,r15 +; ################################ + mulx rbp,rcx,r15 ; a[1]*a[2] adcx r11,rcx adox r12,rbp - mulx rbp,rcx,r8 + mulx rbp,rcx,r8 ; a[1]*a[3] mov rdx,r15 adcx r12,rcx adox r13,rbp adc r13,0 - - mulx r14,rcx,r8 +; ################################ + mulx r14,rcx,r8 ; a[2]*a[3] mov rdx,QWORD[((0+128))+rsi] - xor r15,r15 - adcx r9,r9 + xor r15,r15 ; %r15=0,cf=0,of=0 + adcx r9,r9 ; acc1:6<<1 adox r13,rcx adcx r10,r10 - adox r14,r15 + adox r14,r15 ; of=0 mulx rbp,r8,rdx mov rdx,QWORD[((8+128))+rsi] @@ -2063,7 +2063,7 @@ shrx rax,r8,rsi mov rbp,rdx - +; reduction step 1 add r9,rcx adc r10,rax @@ -2073,7 +2073,7 @@ adc r8,0 shrx rax,r9,rsi - +; reduction step 2 add r10,rcx adc r11,rax @@ -2083,7 +2083,7 @@ adc r9,0 shrx rax,r10,rsi - +; reduction step 3 add r11,rcx adc r8,rax @@ -2093,7 +2093,7 @@ adc r10,0 shrx rax,r11,rsi - +; reduction step 4 add r8,rcx adc r9,rax @@ -2102,7 +2102,7 @@ adc r11,0 xor rdx,rdx - add r12,r8 + add r12,r8 ; accumulate upper half mov rsi,QWORD[(($L$poly+8))] adc r13,r9 mov r8,r12 @@ -2111,12 +2111,12 @@ mov r9,r13 adc rdx,0 - sub r12,-1 + sub r12,-1 ; .Lpoly[0] mov r10,r14 - sbb r13,rsi - sbb r14,0 + sbb r13,rsi ; .Lpoly[1] + sbb r14,0 ; .Lpoly[2] mov r11,r15 - sbb r15,rbp + sbb r15,rbp ; .Lpoly[3] sbb rdx,0 cmovc r12,r8 @@ -2131,8 +2131,8 @@ ret - - +; ############################################################################### +; void ecp_nistz256_select_w5_nohw(uint64_t *val, uint64_t *in_t, int index); global ecp_nistz256_select_w5_nohw ALIGN 32 @@ -2141,17 +2141,17 @@ _CET_ENDBR lea rax,[((-136))+rsp] $L$SEH_begin_ecp_nistz256_select_w5_nohw: - DB 0x48,0x8d,0x60,0xe0 - DB 0x0f,0x29,0x70,0xe0 - DB 0x0f,0x29,0x78,0xf0 - DB 0x44,0x0f,0x29,0x00 - DB 0x44,0x0f,0x29,0x48,0x10 - DB 0x44,0x0f,0x29,0x50,0x20 - DB 0x44,0x0f,0x29,0x58,0x30 - DB 0x44,0x0f,0x29,0x60,0x40 - DB 0x44,0x0f,0x29,0x68,0x50 - DB 0x44,0x0f,0x29,0x70,0x60 - DB 0x44,0x0f,0x29,0x78,0x70 + DB 0x48,0x8d,0x60,0xe0 ; lea -0x20(%rax), %rsp + DB 0x0f,0x29,0x70,0xe0 ; movaps %xmm6, -0x20(%rax) + DB 0x0f,0x29,0x78,0xf0 ; movaps %xmm7, -0x10(%rax) + DB 0x44,0x0f,0x29,0x00 ; movaps %xmm8, 0(%rax) + DB 0x44,0x0f,0x29,0x48,0x10 ; movaps %xmm9, 0x10(%rax) + DB 0x44,0x0f,0x29,0x50,0x20 ; movaps %xmm10, 0x20(%rax) + DB 0x44,0x0f,0x29,0x58,0x30 ; movaps %xmm11, 0x30(%rax) + DB 0x44,0x0f,0x29,0x60,0x40 ; movaps %xmm12, 0x40(%rax) + DB 0x44,0x0f,0x29,0x68,0x50 ; movaps %xmm13, 0x50(%rax) + DB 0x44,0x0f,0x29,0x70,0x60 ; movaps %xmm14, 0x60(%rax) + DB 0x44,0x0f,0x29,0x78,0x70 ; movaps %xmm15, 0x70(%rax) movdqa xmm0,XMMWORD[$L$One] movd xmm1,r8d @@ -2218,8 +2218,8 @@ $L$SEH_end_ecp_nistz256_select_w5_nohw: - - +; ############################################################################### +; void ecp_nistz256_select_w7_nohw(uint64_t *val, uint64_t *in_t, int index); global ecp_nistz256_select_w7_nohw ALIGN 32 @@ -2228,17 +2228,17 @@ _CET_ENDBR lea rax,[((-136))+rsp] $L$SEH_begin_ecp_nistz256_select_w7_nohw: - DB 0x48,0x8d,0x60,0xe0 - DB 0x0f,0x29,0x70,0xe0 - DB 0x0f,0x29,0x78,0xf0 - DB 0x44,0x0f,0x29,0x00 - DB 0x44,0x0f,0x29,0x48,0x10 - DB 0x44,0x0f,0x29,0x50,0x20 - DB 0x44,0x0f,0x29,0x58,0x30 - DB 0x44,0x0f,0x29,0x60,0x40 - DB 0x44,0x0f,0x29,0x68,0x50 - DB 0x44,0x0f,0x29,0x70,0x60 - DB 0x44,0x0f,0x29,0x78,0x70 + DB 0x48,0x8d,0x60,0xe0 ; lea -0x20(%rax), %rsp + DB 0x0f,0x29,0x70,0xe0 ; movaps %xmm6, -0x20(%rax) + DB 0x0f,0x29,0x78,0xf0 ; movaps %xmm7, -0x10(%rax) + DB 0x44,0x0f,0x29,0x00 ; movaps %xmm8, 0(%rax) + DB 0x44,0x0f,0x29,0x48,0x10 ; movaps %xmm9, 0x10(%rax) + DB 0x44,0x0f,0x29,0x50,0x20 ; movaps %xmm10, 0x20(%rax) + DB 0x44,0x0f,0x29,0x58,0x30 ; movaps %xmm11, 0x30(%rax) + DB 0x44,0x0f,0x29,0x60,0x40 ; movaps %xmm12, 0x40(%rax) + DB 0x44,0x0f,0x29,0x68,0x50 ; movaps %xmm13, 0x50(%rax) + DB 0x44,0x0f,0x29,0x70,0x60 ; movaps %xmm14, 0x60(%rax) + DB 0x44,0x0f,0x29,0x78,0x70 ; movaps %xmm15, 0x70(%rax) movdqa xmm8,XMMWORD[$L$One] movd xmm1,r8d @@ -2293,8 +2293,8 @@ $L$SEH_end_ecp_nistz256_select_w7_nohw: - - +; ############################################################################### +; void ecp_nistz256_select_w5_avx2(uint64_t *val, uint64_t *in_t, int index); global ecp_nistz256_select_w5_avx2 ALIGN 32 @@ -2305,17 +2305,17 @@ lea rax,[((-136))+rsp] mov r11,rsp $L$SEH_begin_ecp_nistz256_select_w5_avx2: - DB 0x48,0x8d,0x60,0xe0 - DB 0xc5,0xf8,0x29,0x70,0xe0 - DB 0xc5,0xf8,0x29,0x78,0xf0 - DB 0xc5,0x78,0x29,0x40,0x00 - DB 0xc5,0x78,0x29,0x48,0x10 - DB 0xc5,0x78,0x29,0x50,0x20 - DB 0xc5,0x78,0x29,0x58,0x30 - DB 0xc5,0x78,0x29,0x60,0x40 - DB 0xc5,0x78,0x29,0x68,0x50 - DB 0xc5,0x78,0x29,0x70,0x60 - DB 0xc5,0x78,0x29,0x78,0x70 + DB 0x48,0x8d,0x60,0xe0 ; lea -0x20(%rax), %rsp + DB 0xc5,0xf8,0x29,0x70,0xe0 ; vmovaps %xmm6, -0x20(%rax) + DB 0xc5,0xf8,0x29,0x78,0xf0 ; vmovaps %xmm7, -0x10(%rax) + DB 0xc5,0x78,0x29,0x40,0x00 ; vmovaps %xmm8, 8(%rax) + DB 0xc5,0x78,0x29,0x48,0x10 ; vmovaps %xmm9, 0x10(%rax) + DB 0xc5,0x78,0x29,0x50,0x20 ; vmovaps %xmm10, 0x20(%rax) + DB 0xc5,0x78,0x29,0x58,0x30 ; vmovaps %xmm11, 0x30(%rax) + DB 0xc5,0x78,0x29,0x60,0x40 ; vmovaps %xmm12, 0x40(%rax) + DB 0xc5,0x78,0x29,0x68,0x50 ; vmovaps %xmm13, 0x50(%rax) + DB 0xc5,0x78,0x29,0x70,0x60 ; vmovaps %xmm14, 0x60(%rax) + DB 0xc5,0x78,0x29,0x78,0x70 ; vmovaps %xmm15, 0x70(%rax) vmovdqa ymm0,YMMWORD[$L$Two] vpxor ymm2,ymm2,ymm2 @@ -2383,8 +2383,8 @@ $L$SEH_end_ecp_nistz256_select_w5_avx2: - - +; ############################################################################### +; void ecp_nistz256_select_w7_avx2(uint64_t *val, uint64_t *in_t, int index); global ecp_nistz256_select_w7_avx2 ALIGN 32 @@ -2395,17 +2395,17 @@ mov r11,rsp lea rax,[((-136))+rsp] $L$SEH_begin_ecp_nistz256_select_w7_avx2: - DB 0x48,0x8d,0x60,0xe0 - DB 0xc5,0xf8,0x29,0x70,0xe0 - DB 0xc5,0xf8,0x29,0x78,0xf0 - DB 0xc5,0x78,0x29,0x40,0x00 - DB 0xc5,0x78,0x29,0x48,0x10 - DB 0xc5,0x78,0x29,0x50,0x20 - DB 0xc5,0x78,0x29,0x58,0x30 - DB 0xc5,0x78,0x29,0x60,0x40 - DB 0xc5,0x78,0x29,0x68,0x50 - DB 0xc5,0x78,0x29,0x70,0x60 - DB 0xc5,0x78,0x29,0x78,0x70 + DB 0x48,0x8d,0x60,0xe0 ; lea -0x20(%rax), %rsp + DB 0xc5,0xf8,0x29,0x70,0xe0 ; vmovaps %xmm6, -0x20(%rax) + DB 0xc5,0xf8,0x29,0x78,0xf0 ; vmovaps %xmm7, -0x10(%rax) + DB 0xc5,0x78,0x29,0x40,0x00 ; vmovaps %xmm8, 8(%rax) + DB 0xc5,0x78,0x29,0x48,0x10 ; vmovaps %xmm9, 0x10(%rax) + DB 0xc5,0x78,0x29,0x50,0x20 ; vmovaps %xmm10, 0x20(%rax) + DB 0xc5,0x78,0x29,0x58,0x30 ; vmovaps %xmm11, 0x30(%rax) + DB 0xc5,0x78,0x29,0x60,0x40 ; vmovaps %xmm12, 0x40(%rax) + DB 0xc5,0x78,0x29,0x68,0x50 ; vmovaps %xmm13, 0x50(%rax) + DB 0xc5,0x78,0x29,0x70,0x60 ; vmovaps %xmm14, 0x60(%rax) + DB 0xc5,0x78,0x29,0x78,0x70 ; vmovaps %xmm15, 0x70(%rax) vmovdqa ymm0,YMMWORD[$L$Three] vpxor ymm2,ymm2,ymm2 @@ -2417,7 +2417,7 @@ vmovd xmm1,r8d vpermd ymm1,ymm2,ymm1 - +; Skip index = 0, because it is implicitly the point at infinity mov rax,21 $L$select_loop_avx2_w7: @@ -2588,7 +2588,7 @@ __ecp_nistz256_mul_by_2q: xor r11,r11 - add r12,r12 + add r12,r12 ; a0:a3+a0:a3 adc r13,r13 mov rax,r12 adc r8,r8 @@ -2647,10 +2647,10 @@ $L$point_doubleq_body: $L$point_double_shortcutq: - movdqu xmm0,XMMWORD[rsi] - mov rbx,rsi + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rsi.x + mov rbx,rsi ; backup copy movdqu xmm1,XMMWORD[16+rsi] - mov r12,QWORD[((32+0))+rsi] + mov r12,QWORD[((32+0))+rsi] ; load in_y in "5-4-0-1" order mov r13,QWORD[((32+8))+rsi] mov r8,QWORD[((32+16))+rsi] mov r9,QWORD[((32+24))+rsi] @@ -2665,7 +2665,7 @@ movq xmm2,r11 lea rdi,[rsp] - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_by_2q ; p256_mul_by_2(S, in_y); mov rax,QWORD[((64+0))+rsi] mov r14,QWORD[((64+8))+rsi] @@ -2673,7 +2673,7 @@ mov r8,QWORD[((64+24))+rsi] lea rsi,[((64-0))+rsi] lea rdi,[64+rsp] - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Zsqr, in_z); mov rax,QWORD[((0+0))+rsp] mov r14,QWORD[((8+0))+rsp] @@ -2681,9 +2681,9 @@ mov r15,QWORD[((16+0))+rsp] mov r8,QWORD[((24+0))+rsp] lea rdi,[rsp] - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(S, S); - mov rax,QWORD[32+rbx] + mov rax,QWORD[32+rbx] ; %rbx is still valid mov r9,QWORD[((64+0))+rbx] mov r10,QWORD[((64+8))+rbx] mov r11,QWORD[((64+16))+rbx] @@ -2691,24 +2691,24 @@ lea rsi,[((64-0))+rbx] lea rbx,[32+rbx] movq rdi,xmm2 - call __ecp_nistz256_mul_montq - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_montq ; p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2q ; p256_mul_by_2(res_z, res_z); - mov r12,QWORD[((96+0))+rsp] + mov r12,QWORD[((96+0))+rsp] ; "5-4-0-1" order mov r13,QWORD[((96+8))+rsp] lea rbx,[64+rsp] mov r8,QWORD[((96+16))+rsp] mov r9,QWORD[((96+24))+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_add_toq + call __ecp_nistz256_add_toq ; p256_add(M, in_x, Zsqr); - mov r12,QWORD[((96+0))+rsp] + mov r12,QWORD[((96+0))+rsp] ; "5-4-0-1" order mov r13,QWORD[((96+8))+rsp] lea rbx,[64+rsp] mov r8,QWORD[((96+16))+rsp] mov r9,QWORD[((96+24))+rsp] lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(Zsqr, in_x, Zsqr); mov rax,QWORD[((0+0))+rsp] mov r14,QWORD[((8+0))+rsp] @@ -2716,7 +2716,7 @@ mov r15,QWORD[((16+0))+rsp] mov r8,QWORD[((24+0))+rsp] movq rdi,xmm1 - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(res_y, S); xor r9,r9 mov rax,r12 add r12,-1 @@ -2727,7 +2727,7 @@ mov r8,r15 adc r15,rbp adc r9,0 - xor rsi,rsi + xor rsi,rsi ; borrow %rsi test rax,1 cmovz r12,rax @@ -2736,7 +2736,7 @@ cmovz r15,r8 cmovz r9,rsi - mov rax,r13 + mov rax,r13 ; a0:a3>>1 shr r12,1 shl rax,63 mov r10,r14 @@ -2763,14 +2763,14 @@ mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(M, M, Zsqr); lea rdi,[128+rsp] call __ecp_nistz256_mul_by_2q lea rbx,[32+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_add_toq + call __ecp_nistz256_add_toq ; p256_mul_by_3(M, M); mov rax,QWORD[96+rsp] lea rbx,[96+rsp] @@ -2780,10 +2780,10 @@ mov r11,QWORD[((16+0))+rsp] mov r12,QWORD[((24+0))+rsp] lea rdi,[rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(S, S, in_x); lea rdi,[128+rsp] - call __ecp_nistz256_mul_by_2q + call __ecp_nistz256_mul_by_2q ; p256_mul_by_2(tmp0, S); mov rax,QWORD[((0+32))+rsp] mov r14,QWORD[((8+32))+rsp] @@ -2791,27 +2791,27 @@ mov r15,QWORD[((16+32))+rsp] mov r8,QWORD[((24+32))+rsp] movq rdi,xmm0 - call __ecp_nistz256_sqr_montq + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(res_x, M); lea rbx,[128+rsp] - mov r8,r14 + mov r8,r14 ; harmonize sqr output and sub input mov r9,r15 mov r14,rsi mov r15,rbp - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(res_x, res_x, tmp0); mov rax,QWORD[((0+0))+rsp] mov rbp,QWORD[((0+8))+rsp] mov rcx,QWORD[((0+16))+rsp] - mov r10,QWORD[((0+24))+rsp] + mov r10,QWORD[((0+24))+rsp] ; "4-5-0-1" order lea rdi,[rsp] - call __ecp_nistz256_subq + call __ecp_nistz256_subq ; p256_sub(S, S, res_x); mov rax,QWORD[32+rsp] lea rbx,[32+rsp] - mov r14,r12 + mov r14,r12 ; harmonize sub output and mul input xor ecx,ecx - mov QWORD[((0+0))+rsp],r12 + mov QWORD[((0+0))+rsp],r12 ; have to save:-( mov r10,r13 mov QWORD[((0+8))+rsp],r13 cmovz r11,r8 @@ -2821,11 +2821,11 @@ mov QWORD[((0+24))+rsp],r9 mov r9,r14 lea rdi,[rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(S, S, M); movq rbx,xmm1 movq rdi,xmm1 - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(res_y, S, res_y); lea rsi,[((160+56))+rsp] @@ -2880,14 +2880,14 @@ $L$point_addq_body: - movdqu xmm0,XMMWORD[rsi] + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rsi movdqu xmm1,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm3,XMMWORD[48+rsi] movdqu xmm4,XMMWORD[64+rsi] movdqu xmm5,XMMWORD[80+rsi] - mov rbx,rsi - mov rsi,rdx + mov rbx,rsi ; reassign + mov rsi,rdx ; reassign movdqa XMMWORD[384+rsp],xmm0 movdqa XMMWORD[(384+16)+rsp],xmm1 movdqa XMMWORD[416+rsp],xmm2 @@ -2896,54 +2896,54 @@ movdqa XMMWORD[(448+16)+rsp],xmm5 por xmm5,xmm4 - movdqu xmm0,XMMWORD[rsi] + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rbx pshufd xmm3,xmm5,0xb1 movdqu xmm1,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] por xmm5,xmm3 movdqu xmm3,XMMWORD[48+rsi] - mov rax,QWORD[((64+0))+rsi] + mov rax,QWORD[((64+0))+rsi] ; load original in2_z mov r14,QWORD[((64+8))+rsi] mov r15,QWORD[((64+16))+rsi] mov r8,QWORD[((64+24))+rsi] movdqa XMMWORD[480+rsp],xmm0 pshufd xmm4,xmm5,0x1e movdqa XMMWORD[(480+16)+rsp],xmm1 - movdqu xmm0,XMMWORD[64+rsi] + movdqu xmm0,XMMWORD[64+rsi] ; in2_z again movdqu xmm1,XMMWORD[80+rsi] movdqa XMMWORD[512+rsp],xmm2 movdqa XMMWORD[(512+16)+rsp],xmm3 por xmm5,xmm4 pxor xmm4,xmm4 por xmm1,xmm0 - movq xmm0,rdi + movq xmm0,rdi ; save %rdi - lea rsi,[((64-0))+rsi] - mov QWORD[((544+0))+rsp],rax + lea rsi,[((64-0))+rsi] ; %rsi is still valid + mov QWORD[((544+0))+rsp],rax ; make in2_z copy mov QWORD[((544+8))+rsp],r14 mov QWORD[((544+16))+rsp],r15 mov QWORD[((544+24))+rsp],r8 - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montq + lea rdi,[96+rsp] ; Z2^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Z2sqr, in2_z); pcmpeqd xmm5,xmm4 pshufd xmm4,xmm1,0xb1 por xmm4,xmm1 - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; in1infty pshufd xmm3,xmm4,0x1e por xmm4,xmm3 pxor xmm3,xmm3 pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 - mov rax,QWORD[((64+0))+rbx] + pshufd xmm4,xmm4,0 ; in2infty + mov rax,QWORD[((64+0))+rbx] ; load original in1_z mov r14,QWORD[((64+8))+rbx] mov r15,QWORD[((64+16))+rbx] mov r8,QWORD[((64+24))+rbx] movq xmm1,rbx lea rsi,[((64-0))+rbx] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montq + lea rdi,[32+rsp] ; Z1^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Z1sqr, in1_z); mov rax,QWORD[544+rsp] lea rbx,[544+rsp] @@ -2952,8 +2952,8 @@ lea rsi,[((0+96))+rsp] mov r11,QWORD[((16+96))+rsp] mov r12,QWORD[((24+96))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[224+rsp] ; S1 = Z2^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(S1, Z2sqr, in2_z); mov rax,QWORD[448+rsp] lea rbx,[448+rsp] @@ -2962,8 +2962,8 @@ lea rsi,[((0+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[256+rsp] ; S2 = Z1^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(S2, Z1sqr, in1_z); mov rax,QWORD[416+rsp] lea rbx,[416+rsp] @@ -2972,8 +2972,8 @@ lea rsi,[((0+224))+rsp] mov r11,QWORD[((16+224))+rsp] mov r12,QWORD[((24+224))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[224+rsp] ; S1 = Y1*Z2^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(S1, S1, in1_y); mov rax,QWORD[512+rsp] lea rbx,[512+rsp] @@ -2982,18 +2982,18 @@ lea rsi,[((0+256))+rsp] mov r11,QWORD[((16+256))+rsp] mov r12,QWORD[((24+256))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[256+rsp] ; S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(S2, S2, in2_y); lea rbx,[224+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromq + lea rdi,[64+rsp] ; R = S2 - S1 + call __ecp_nistz256_sub_fromq ; p256_sub(R, S2, S1); - or r12,r13 + or r12,r13 ; see if result is zero movdqa xmm2,xmm4 or r12,r8 or r12,r9 - por xmm2,xmm5 + por xmm2,xmm5 ; in1infty || in2infty movq xmm3,r12 mov rax,QWORD[384+rsp] @@ -3003,8 +3003,8 @@ lea rsi,[((0+96))+rsp] mov r11,QWORD[((16+96))+rsp] mov r12,QWORD[((24+96))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[160+rsp] ; U1 = X1*Z2^2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(U1, in1_x, Z2sqr); mov rax,QWORD[480+rsp] lea rbx,[480+rsp] @@ -3013,34 +3013,34 @@ lea rsi,[((0+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[192+rsp] ; U2 = X2*Z1^2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(U2, in2_x, Z1sqr); lea rbx,[160+rsp] - lea rdi,[rsp] - call __ecp_nistz256_sub_fromq + lea rdi,[rsp] ; H = U2 - U1 + call __ecp_nistz256_sub_fromq ; p256_sub(H, U2, U1); - or r12,r13 + or r12,r13 ; see if result is zero or r12,r8 - or r12,r9 + or r12,r9 ; !is_equal(U1, U2) movq r8,xmm2 movq r9,xmm3 or r12,r8 - DB 0x3e - jnz NEAR $L$add_proceedq + DB 0x3e ; predict taken + jnz NEAR $L$add_proceedq ; !is_equal(U1, U2) || in1infty || in2infty - - +; We now know A = B or A = -B and neither is infinity. Compare the +; y-coordinates via S1 and S2. test r9,r9 - jz NEAR $L$add_doubleq + jz NEAR $L$add_doubleq ; is_equal(S1, S2) - - - - - - movq rdi,xmm0 +; A = -B, so the result is infinity. +; +; TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in +; which case we should eliminate this special-case and simplify the +; timing analysis. + movq rdi,xmm0 ; restore %rdi pxor xmm0,xmm0 movdqu XMMWORD[rdi],xmm0 movdqu XMMWORD[16+rdi],xmm0 @@ -3052,9 +3052,9 @@ ALIGN 32 $L$add_doubleq: - movq rsi,xmm1 - movq rdi,xmm0 - add rsp,416 + movq rsi,xmm1 ; restore %rsi + movq rdi,xmm0 ; restore %rdi + add rsp,416 ; difference in frame sizes jmp NEAR $L$point_double_shortcutq @@ -3066,8 +3066,8 @@ lea rsi,[((0+64))+rsp] mov r15,QWORD[((16+64))+rsp] mov r8,QWORD[((24+64))+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montq + lea rdi,[96+rsp] ; R^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Rsqr, R); mov rax,QWORD[448+rsp] lea rbx,[448+rsp] @@ -3076,16 +3076,16 @@ lea rsi,[((0+0))+rsp] mov r11,QWORD[((16+0))+rsp] mov r12,QWORD[((24+0))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[352+rsp] ; Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(res_z, H, in1_z); mov rax,QWORD[((0+0))+rsp] mov r14,QWORD[((8+0))+rsp] lea rsi,[((0+0))+rsp] mov r15,QWORD[((16+0))+rsp] mov r8,QWORD[((24+0))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montq + lea rdi,[32+rsp] ; H^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Hsqr, H); mov rax,QWORD[544+rsp] lea rbx,[544+rsp] @@ -3094,8 +3094,8 @@ lea rsi,[((0+352))+rsp] mov r11,QWORD[((16+352))+rsp] mov r12,QWORD[((24+352))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[352+rsp] ; Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(res_z, res_z, in2_z); mov rax,QWORD[rsp] lea rbx,[rsp] @@ -3104,8 +3104,8 @@ lea rsi,[((0+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[128+rsp] ; H^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(Hcub, Hsqr, H); mov rax,QWORD[160+rsp] lea rbx,[160+rsp] @@ -3114,14 +3114,14 @@ lea rsi,[((0+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montq - - - + lea rdi,[192+rsp] ; U1*H^2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(U2, U1, Hsqr); +; lea 192(%rsp), %rsi +; lea 32(%rsp), %rdi # 2*U1*H^2 +; call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xor r11,r11 - add r12,r12 + add r12,r12 ; a0:a3+a0:a3 lea rsi,[96+rsp] adc r13,r13 mov rax,r12 @@ -3147,11 +3147,11 @@ cmovc r9,r10 mov r10,QWORD[24+rsi] - call __ecp_nistz256_subq + call __ecp_nistz256_subq ; p256_sub(res_x, Rsqr, Hsqr); lea rbx,[128+rsp] lea rdi,[288+rsp] - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(res_x, res_x, Hcub); mov rax,QWORD[((192+0))+rsp] mov rbp,QWORD[((192+8))+rsp] @@ -3159,10 +3159,10 @@ mov r10,QWORD[((192+24))+rsp] lea rdi,[320+rsp] - call __ecp_nistz256_subq + call __ecp_nistz256_subq ; p256_sub(res_y, U2, res_x); - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 + mov QWORD[rdi],r12 ; save the result, as + mov QWORD[8+rdi],r13 ; __ecp_nistz256_sub doesn't mov QWORD[16+rdi],r8 mov QWORD[24+rdi],r9 mov rax,QWORD[128+rsp] @@ -3173,7 +3173,7 @@ mov r11,QWORD[((16+224))+rsp] mov r12,QWORD[((24+224))+rsp] lea rdi,[256+rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(S2, S1, Hcub); mov rax,QWORD[320+rsp] lea rbx,[320+rsp] @@ -3183,15 +3183,15 @@ mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] lea rdi,[320+rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(res_y, R, res_y); lea rbx,[256+rsp] lea rdi,[320+rsp] - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(res_y, res_y, S2); - movq rdi,xmm0 + movq rdi,xmm0 ; restore %rdi - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_z, in2_z, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[352+rsp] movdqa xmm2,xmm5 @@ -3202,7 +3202,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_z, in1_z, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -3215,7 +3215,7 @@ movdqu XMMWORD[64+rdi],xmm2 movdqu XMMWORD[80+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_x, in2_x, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[288+rsp] movdqa xmm2,xmm5 @@ -3226,7 +3226,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_x, in1_x, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -3239,7 +3239,7 @@ movdqu XMMWORD[rdi],xmm2 movdqu XMMWORD[16+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_y, in2_y, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[320+rsp] movdqa xmm2,xmm5 @@ -3250,7 +3250,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_y, in1_y, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -3317,14 +3317,14 @@ $L$add_affineq_body: - movdqu xmm0,XMMWORD[rsi] - mov rbx,rdx + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rsi + mov rbx,rdx ; reassign movdqu xmm1,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm3,XMMWORD[48+rsi] movdqu xmm4,XMMWORD[64+rsi] movdqu xmm5,XMMWORD[80+rsi] - mov rax,QWORD[((64+0))+rsi] + mov rax,QWORD[((64+0))+rsi] ; load original in1_z mov r14,QWORD[((64+8))+rsi] mov r15,QWORD[((64+16))+rsi] mov r8,QWORD[((64+24))+rsi] @@ -3336,7 +3336,7 @@ movdqa XMMWORD[(384+16)+rsp],xmm5 por xmm5,xmm4 - movdqu xmm0,XMMWORD[rbx] + movdqu xmm0,XMMWORD[rbx] ; copy *(P256_POINT_AFFINE *)%rbx pshufd xmm3,xmm5,0xb1 movdqu xmm1,XMMWORD[16+rbx] movdqu xmm2,XMMWORD[32+rbx] @@ -3346,7 +3346,7 @@ pshufd xmm4,xmm5,0x1e movdqa XMMWORD[(416+16)+rsp],xmm1 por xmm1,xmm0 - movq xmm0,rdi + movq xmm0,rdi ; save %rdi movdqa XMMWORD[448+rsp],xmm2 movdqa XMMWORD[(448+16)+rsp],xmm3 por xmm3,xmm2 @@ -3354,33 +3354,33 @@ pxor xmm4,xmm4 por xmm3,xmm1 - lea rsi,[((64-0))+rsi] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montq + lea rsi,[((64-0))+rsi] ; %rsi is still valid + lea rdi,[32+rsp] ; Z1^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Z1sqr, in1_z); pcmpeqd xmm5,xmm4 pshufd xmm4,xmm3,0xb1 - mov rax,QWORD[rbx] - - mov r9,r12 + mov rax,QWORD[rbx] ; %rbx is still valid +; lea 0x00(%rbx), %rbx + mov r9,r12 ; harmonize sqr output and mul input por xmm4,xmm3 - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; in1infty pshufd xmm3,xmm4,0x1e mov r10,r13 por xmm4,xmm3 pxor xmm3,xmm3 mov r11,r14 pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 + pshufd xmm4,xmm4,0 ; in2infty lea rsi,[((32-0))+rsp] mov r12,r15 - lea rdi,[rsp] - call __ecp_nistz256_mul_montq + lea rdi,[rsp] ; U2 = X2*Z1^2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(U2, Z1sqr, in2_x); lea rbx,[320+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromq + lea rdi,[64+rsp] ; H = U2 - U1 + call __ecp_nistz256_sub_fromq ; p256_sub(H, U2, in1_x); mov rax,QWORD[384+rsp] lea rbx,[384+rsp] @@ -3389,8 +3389,8 @@ lea rsi,[((0+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[32+rsp] ; S2 = Z1^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(S2, Z1sqr, in1_z); mov rax,QWORD[384+rsp] lea rbx,[384+rsp] @@ -3399,8 +3399,8 @@ lea rsi,[((0+64))+rsp] mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] - lea rdi,[288+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[288+rsp] ; Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(res_z, H, in1_z); mov rax,QWORD[448+rsp] lea rbx,[448+rsp] @@ -3409,28 +3409,28 @@ lea rsi,[((0+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[32+rsp] ; S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(S2, S2, in2_y); lea rbx,[352+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sub_fromq + lea rdi,[96+rsp] ; R = S2 - S1 + call __ecp_nistz256_sub_fromq ; p256_sub(R, S2, in1_y); mov rax,QWORD[((0+64))+rsp] mov r14,QWORD[((8+64))+rsp] lea rsi,[((0+64))+rsp] mov r15,QWORD[((16+64))+rsp] mov r8,QWORD[((24+64))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_sqr_montq + lea rdi,[128+rsp] ; H^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Hsqr, H); mov rax,QWORD[((0+96))+rsp] mov r14,QWORD[((8+96))+rsp] lea rsi,[((0+96))+rsp] mov r15,QWORD[((16+96))+rsp] mov r8,QWORD[((24+96))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_sqr_montq + lea rdi,[192+rsp] ; R^2 + call __ecp_nistz256_sqr_montq ; p256_sqr_mont(Rsqr, R); mov rax,QWORD[128+rsp] lea rbx,[128+rsp] @@ -3439,8 +3439,8 @@ lea rsi,[((0+64))+rsp] mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montq + lea rdi,[160+rsp] ; H^3 + call __ecp_nistz256_mul_montq ; p256_mul_mont(Hcub, Hsqr, H); mov rax,QWORD[320+rsp] lea rbx,[320+rsp] @@ -3449,14 +3449,14 @@ lea rsi,[((0+128))+rsp] mov r11,QWORD[((16+128))+rsp] mov r12,QWORD[((24+128))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_mul_montq - - - + lea rdi,[rsp] ; U1*H^2 + call __ecp_nistz256_mul_montq ; p256_mul_mont(U2, in1_x, Hsqr); +; lea 0(%rsp), %rsi +; lea 128(%rsp), %rdi # 2*U1*H^2 +; call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xor r11,r11 - add r12,r12 + add r12,r12 ; a0:a3+a0:a3 lea rsi,[192+rsp] adc r13,r13 mov rax,r12 @@ -3482,11 +3482,11 @@ cmovc r9,r10 mov r10,QWORD[24+rsi] - call __ecp_nistz256_subq + call __ecp_nistz256_subq ; p256_sub(res_x, Rsqr, Hsqr); lea rbx,[160+rsp] lea rdi,[224+rsp] - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(res_x, res_x, Hcub); mov rax,QWORD[((0+0))+rsp] mov rbp,QWORD[((0+8))+rsp] @@ -3494,10 +3494,10 @@ mov r10,QWORD[((0+24))+rsp] lea rdi,[64+rsp] - call __ecp_nistz256_subq + call __ecp_nistz256_subq ; p256_sub(H, U2, res_x); - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 + mov QWORD[rdi],r12 ; save the result, as + mov QWORD[8+rdi],r13 ; __ecp_nistz256_sub doesn't mov QWORD[16+rdi],r8 mov QWORD[24+rdi],r9 mov rax,QWORD[352+rsp] @@ -3508,7 +3508,7 @@ mov r11,QWORD[((16+160))+rsp] mov r12,QWORD[((24+160))+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(S2, Hcub, in1_y); mov rax,QWORD[96+rsp] lea rbx,[96+rsp] @@ -3518,15 +3518,15 @@ mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] lea rdi,[64+rsp] - call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_montq ; p256_mul_mont(H, H, R); lea rbx,[32+rsp] lea rdi,[256+rsp] - call __ecp_nistz256_sub_fromq + call __ecp_nistz256_sub_fromq ; p256_sub(res_y, H, S2); - movq rdi,xmm0 + movq rdi,xmm0 ; restore %rdi - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_z, ONE, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[288+rsp] movdqa xmm2,xmm5 @@ -3537,7 +3537,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_z, in1_z, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -3550,7 +3550,7 @@ movdqu XMMWORD[64+rdi],xmm2 movdqu XMMWORD[80+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_x, in2_x, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[224+rsp] movdqa xmm2,xmm5 @@ -3561,7 +3561,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_x, in1_x, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -3574,7 +3574,7 @@ movdqu XMMWORD[rdi],xmm2 movdqu XMMWORD[16+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_y, in2_y, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[256+rsp] movdqa xmm2,xmm5 @@ -3585,7 +3585,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_y, in1_y, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -3726,7 +3726,7 @@ __ecp_nistz256_mul_by_2x: xor r11,r11 - adc r12,r12 + adc r12,r12 ; a0:a3+a0:a3 adc r13,r13 mov rax,r12 adc r8,r8 @@ -3786,10 +3786,10 @@ $L$point_doublex_body: $L$point_double_shortcutx: - movdqu xmm0,XMMWORD[rsi] - mov rbx,rsi + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rsi.x + mov rbx,rsi ; backup copy movdqu xmm1,XMMWORD[16+rsi] - mov r12,QWORD[((32+0))+rsi] + mov r12,QWORD[((32+0))+rsi] ; load in_y in "5-4-0-1" order mov r13,QWORD[((32+8))+rsi] mov r8,QWORD[((32+16))+rsi] mov r9,QWORD[((32+24))+rsi] @@ -3804,7 +3804,7 @@ movq xmm2,r11 lea rdi,[rsp] - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_by_2x ; p256_mul_by_2(S, in_y); mov rdx,QWORD[((64+0))+rsi] mov r14,QWORD[((64+8))+rsi] @@ -3812,7 +3812,7 @@ mov r8,QWORD[((64+24))+rsi] lea rsi,[((64-128))+rsi] lea rdi,[64+rsp] - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Zsqr, in_z); mov rdx,QWORD[((0+0))+rsp] mov r14,QWORD[((8+0))+rsp] @@ -3820,9 +3820,9 @@ mov r15,QWORD[((16+0))+rsp] mov r8,QWORD[((24+0))+rsp] lea rdi,[rsp] - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(S, S); - mov rdx,QWORD[32+rbx] + mov rdx,QWORD[32+rbx] ; %rbx is still valid mov r9,QWORD[((64+0))+rbx] mov r10,QWORD[((64+8))+rbx] mov r11,QWORD[((64+16))+rbx] @@ -3830,24 +3830,24 @@ lea rsi,[((64-128))+rbx] lea rbx,[32+rbx] movq rdi,xmm2 - call __ecp_nistz256_mul_montx - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_montx ; p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2x ; p256_mul_by_2(res_z, res_z); - mov r12,QWORD[((96+0))+rsp] + mov r12,QWORD[((96+0))+rsp] ; "5-4-0-1" order mov r13,QWORD[((96+8))+rsp] lea rbx,[64+rsp] mov r8,QWORD[((96+16))+rsp] mov r9,QWORD[((96+24))+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_add_tox + call __ecp_nistz256_add_tox ; p256_add(M, in_x, Zsqr); - mov r12,QWORD[((96+0))+rsp] + mov r12,QWORD[((96+0))+rsp] ; "5-4-0-1" order mov r13,QWORD[((96+8))+rsp] lea rbx,[64+rsp] mov r8,QWORD[((96+16))+rsp] mov r9,QWORD[((96+24))+rsp] lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(Zsqr, in_x, Zsqr); mov rdx,QWORD[((0+0))+rsp] mov r14,QWORD[((8+0))+rsp] @@ -3855,7 +3855,7 @@ mov r15,QWORD[((16+0))+rsp] mov r8,QWORD[((24+0))+rsp] movq rdi,xmm1 - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(res_y, S); xor r9,r9 mov rax,r12 add r12,-1 @@ -3866,7 +3866,7 @@ mov r8,r15 adc r15,rbp adc r9,0 - xor rsi,rsi + xor rsi,rsi ; borrow %rsi test rax,1 cmovz r12,rax @@ -3875,7 +3875,7 @@ cmovz r15,r8 cmovz r9,rsi - mov rax,r13 + mov rax,r13 ; a0:a3>>1 shr r12,1 shl rax,63 mov r10,r14 @@ -3902,14 +3902,14 @@ mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(M, M, Zsqr); lea rdi,[128+rsp] call __ecp_nistz256_mul_by_2x lea rbx,[32+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_add_tox + call __ecp_nistz256_add_tox ; p256_mul_by_3(M, M); mov rdx,QWORD[96+rsp] lea rbx,[96+rsp] @@ -3919,10 +3919,10 @@ mov r11,QWORD[((16+0))+rsp] mov r12,QWORD[((24+0))+rsp] lea rdi,[rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(S, S, in_x); lea rdi,[128+rsp] - call __ecp_nistz256_mul_by_2x + call __ecp_nistz256_mul_by_2x ; p256_mul_by_2(tmp0, S); mov rdx,QWORD[((0+32))+rsp] mov r14,QWORD[((8+32))+rsp] @@ -3930,27 +3930,27 @@ mov r15,QWORD[((16+32))+rsp] mov r8,QWORD[((24+32))+rsp] movq rdi,xmm0 - call __ecp_nistz256_sqr_montx + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(res_x, M); lea rbx,[128+rsp] - mov r8,r14 + mov r8,r14 ; harmonize sqr output and sub input mov r9,r15 mov r14,rsi mov r15,rbp - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(res_x, res_x, tmp0); mov rax,QWORD[((0+0))+rsp] mov rbp,QWORD[((0+8))+rsp] mov rcx,QWORD[((0+16))+rsp] - mov r10,QWORD[((0+24))+rsp] + mov r10,QWORD[((0+24))+rsp] ; "4-5-0-1" order lea rdi,[rsp] - call __ecp_nistz256_subx + call __ecp_nistz256_subx ; p256_sub(S, S, res_x); mov rdx,QWORD[32+rsp] lea rbx,[32+rsp] - mov r14,r12 + mov r14,r12 ; harmonize sub output and mul input xor ecx,ecx - mov QWORD[((0+0))+rsp],r12 + mov QWORD[((0+0))+rsp],r12 ; have to save:-( mov r10,r13 mov QWORD[((0+8))+rsp],r13 cmovz r11,r8 @@ -3960,11 +3960,11 @@ mov QWORD[((0+24))+rsp],r9 mov r9,r14 lea rdi,[rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(S, S, M); movq rbx,xmm1 movq rdi,xmm1 - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(res_y, S, res_y); lea rsi,[((160+56))+rsp] @@ -4019,14 +4019,14 @@ $L$point_addx_body: - movdqu xmm0,XMMWORD[rsi] + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rsi movdqu xmm1,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm3,XMMWORD[48+rsi] movdqu xmm4,XMMWORD[64+rsi] movdqu xmm5,XMMWORD[80+rsi] - mov rbx,rsi - mov rsi,rdx + mov rbx,rsi ; reassign + mov rsi,rdx ; reassign movdqa XMMWORD[384+rsp],xmm0 movdqa XMMWORD[(384+16)+rsp],xmm1 movdqa XMMWORD[416+rsp],xmm2 @@ -4035,54 +4035,54 @@ movdqa XMMWORD[(448+16)+rsp],xmm5 por xmm5,xmm4 - movdqu xmm0,XMMWORD[rsi] + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rbx pshufd xmm3,xmm5,0xb1 movdqu xmm1,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] por xmm5,xmm3 movdqu xmm3,XMMWORD[48+rsi] - mov rdx,QWORD[((64+0))+rsi] + mov rdx,QWORD[((64+0))+rsi] ; load original in2_z mov r14,QWORD[((64+8))+rsi] mov r15,QWORD[((64+16))+rsi] mov r8,QWORD[((64+24))+rsi] movdqa XMMWORD[480+rsp],xmm0 pshufd xmm4,xmm5,0x1e movdqa XMMWORD[(480+16)+rsp],xmm1 - movdqu xmm0,XMMWORD[64+rsi] + movdqu xmm0,XMMWORD[64+rsi] ; in2_z again movdqu xmm1,XMMWORD[80+rsi] movdqa XMMWORD[512+rsp],xmm2 movdqa XMMWORD[(512+16)+rsp],xmm3 por xmm5,xmm4 pxor xmm4,xmm4 por xmm1,xmm0 - movq xmm0,rdi + movq xmm0,rdi ; save %rdi - lea rsi,[((64-128))+rsi] - mov QWORD[((544+0))+rsp],rdx + lea rsi,[((64-128))+rsi] ; %rsi is still valid + mov QWORD[((544+0))+rsp],rdx ; make in2_z copy mov QWORD[((544+8))+rsp],r14 mov QWORD[((544+16))+rsp],r15 mov QWORD[((544+24))+rsp],r8 - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montx + lea rdi,[96+rsp] ; Z2^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Z2sqr, in2_z); pcmpeqd xmm5,xmm4 pshufd xmm4,xmm1,0xb1 por xmm4,xmm1 - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; in1infty pshufd xmm3,xmm4,0x1e por xmm4,xmm3 pxor xmm3,xmm3 pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 - mov rdx,QWORD[((64+0))+rbx] + pshufd xmm4,xmm4,0 ; in2infty + mov rdx,QWORD[((64+0))+rbx] ; load original in1_z mov r14,QWORD[((64+8))+rbx] mov r15,QWORD[((64+16))+rbx] mov r8,QWORD[((64+24))+rbx] movq xmm1,rbx lea rsi,[((64-128))+rbx] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montx + lea rdi,[32+rsp] ; Z1^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Z1sqr, in1_z); mov rdx,QWORD[544+rsp] lea rbx,[544+rsp] @@ -4091,8 +4091,8 @@ lea rsi,[((-128+96))+rsp] mov r11,QWORD[((16+96))+rsp] mov r12,QWORD[((24+96))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[224+rsp] ; S1 = Z2^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(S1, Z2sqr, in2_z); mov rdx,QWORD[448+rsp] lea rbx,[448+rsp] @@ -4101,8 +4101,8 @@ lea rsi,[((-128+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[256+rsp] ; S2 = Z1^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(S2, Z1sqr, in1_z); mov rdx,QWORD[416+rsp] lea rbx,[416+rsp] @@ -4111,8 +4111,8 @@ lea rsi,[((-128+224))+rsp] mov r11,QWORD[((16+224))+rsp] mov r12,QWORD[((24+224))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[224+rsp] ; S1 = Y1*Z2^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(S1, S1, in1_y); mov rdx,QWORD[512+rsp] lea rbx,[512+rsp] @@ -4121,18 +4121,18 @@ lea rsi,[((-128+256))+rsp] mov r11,QWORD[((16+256))+rsp] mov r12,QWORD[((24+256))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[256+rsp] ; S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(S2, S2, in2_y); lea rbx,[224+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromx + lea rdi,[64+rsp] ; R = S2 - S1 + call __ecp_nistz256_sub_fromx ; p256_sub(R, S2, S1); - or r12,r13 + or r12,r13 ; see if result is zero movdqa xmm2,xmm4 or r12,r8 or r12,r9 - por xmm2,xmm5 + por xmm2,xmm5 ; in1infty || in2infty movq xmm3,r12 mov rdx,QWORD[384+rsp] @@ -4142,8 +4142,8 @@ lea rsi,[((-128+96))+rsp] mov r11,QWORD[((16+96))+rsp] mov r12,QWORD[((24+96))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[160+rsp] ; U1 = X1*Z2^2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(U1, in1_x, Z2sqr); mov rdx,QWORD[480+rsp] lea rbx,[480+rsp] @@ -4152,34 +4152,34 @@ lea rsi,[((-128+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[192+rsp] ; U2 = X2*Z1^2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(U2, in2_x, Z1sqr); lea rbx,[160+rsp] - lea rdi,[rsp] - call __ecp_nistz256_sub_fromx + lea rdi,[rsp] ; H = U2 - U1 + call __ecp_nistz256_sub_fromx ; p256_sub(H, U2, U1); - or r12,r13 + or r12,r13 ; see if result is zero or r12,r8 - or r12,r9 + or r12,r9 ; !is_equal(U1, U2) movq r8,xmm2 movq r9,xmm3 or r12,r8 - DB 0x3e - jnz NEAR $L$add_proceedx + DB 0x3e ; predict taken + jnz NEAR $L$add_proceedx ; !is_equal(U1, U2) || in1infty || in2infty - - +; We now know A = B or A = -B and neither is infinity. Compare the +; y-coordinates via S1 and S2. test r9,r9 - jz NEAR $L$add_doublex + jz NEAR $L$add_doublex ; is_equal(S1, S2) - - - - - - movq rdi,xmm0 +; A = -B, so the result is infinity. +; +; TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in +; which case we should eliminate this special-case and simplify the +; timing analysis. + movq rdi,xmm0 ; restore %rdi pxor xmm0,xmm0 movdqu XMMWORD[rdi],xmm0 movdqu XMMWORD[16+rdi],xmm0 @@ -4191,9 +4191,9 @@ ALIGN 32 $L$add_doublex: - movq rsi,xmm1 - movq rdi,xmm0 - add rsp,416 + movq rsi,xmm1 ; restore %rsi + movq rdi,xmm0 ; restore %rdi + add rsp,416 ; difference in frame sizes jmp NEAR $L$point_double_shortcutx @@ -4205,8 +4205,8 @@ lea rsi,[((-128+64))+rsp] mov r15,QWORD[((16+64))+rsp] mov r8,QWORD[((24+64))+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montx + lea rdi,[96+rsp] ; R^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Rsqr, R); mov rdx,QWORD[448+rsp] lea rbx,[448+rsp] @@ -4215,16 +4215,16 @@ lea rsi,[((-128+0))+rsp] mov r11,QWORD[((16+0))+rsp] mov r12,QWORD[((24+0))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[352+rsp] ; Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(res_z, H, in1_z); mov rdx,QWORD[((0+0))+rsp] mov r14,QWORD[((8+0))+rsp] lea rsi,[((-128+0))+rsp] mov r15,QWORD[((16+0))+rsp] mov r8,QWORD[((24+0))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montx + lea rdi,[32+rsp] ; H^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Hsqr, H); mov rdx,QWORD[544+rsp] lea rbx,[544+rsp] @@ -4233,8 +4233,8 @@ lea rsi,[((-128+352))+rsp] mov r11,QWORD[((16+352))+rsp] mov r12,QWORD[((24+352))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[352+rsp] ; Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(res_z, res_z, in2_z); mov rdx,QWORD[rsp] lea rbx,[rsp] @@ -4243,8 +4243,8 @@ lea rsi,[((-128+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[128+rsp] ; H^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(Hcub, Hsqr, H); mov rdx,QWORD[160+rsp] lea rbx,[160+rsp] @@ -4253,14 +4253,14 @@ lea rsi,[((-128+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montx - - - + lea rdi,[192+rsp] ; U1*H^2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(U2, U1, Hsqr); +; lea 192(%rsp), %rsi +; lea 32(%rsp), %rdi # 2*U1*H^2 +; call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xor r11,r11 - add r12,r12 + add r12,r12 ; a0:a3+a0:a3 lea rsi,[96+rsp] adc r13,r13 mov rax,r12 @@ -4286,11 +4286,11 @@ cmovc r9,r10 mov r10,QWORD[24+rsi] - call __ecp_nistz256_subx + call __ecp_nistz256_subx ; p256_sub(res_x, Rsqr, Hsqr); lea rbx,[128+rsp] lea rdi,[288+rsp] - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(res_x, res_x, Hcub); mov rax,QWORD[((192+0))+rsp] mov rbp,QWORD[((192+8))+rsp] @@ -4298,10 +4298,10 @@ mov r10,QWORD[((192+24))+rsp] lea rdi,[320+rsp] - call __ecp_nistz256_subx + call __ecp_nistz256_subx ; p256_sub(res_y, U2, res_x); - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 + mov QWORD[rdi],r12 ; save the result, as + mov QWORD[8+rdi],r13 ; __ecp_nistz256_sub doesn't mov QWORD[16+rdi],r8 mov QWORD[24+rdi],r9 mov rdx,QWORD[128+rsp] @@ -4312,7 +4312,7 @@ mov r11,QWORD[((16+224))+rsp] mov r12,QWORD[((24+224))+rsp] lea rdi,[256+rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(S2, S1, Hcub); mov rdx,QWORD[320+rsp] lea rbx,[320+rsp] @@ -4322,15 +4322,15 @@ mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] lea rdi,[320+rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(res_y, R, res_y); lea rbx,[256+rsp] lea rdi,[320+rsp] - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(res_y, res_y, S2); - movq rdi,xmm0 + movq rdi,xmm0 ; restore %rdi - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_z, in2_z, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[352+rsp] movdqa xmm2,xmm5 @@ -4341,7 +4341,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_z, in1_z, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -4354,7 +4354,7 @@ movdqu XMMWORD[64+rdi],xmm2 movdqu XMMWORD[80+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_x, in2_x, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[288+rsp] movdqa xmm2,xmm5 @@ -4365,7 +4365,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_x, in1_x, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -4378,7 +4378,7 @@ movdqu XMMWORD[rdi],xmm2 movdqu XMMWORD[16+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_y, in2_y, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[320+rsp] movdqa xmm2,xmm5 @@ -4389,7 +4389,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_y, in1_y, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -4456,14 +4456,14 @@ $L$add_affinex_body: - movdqu xmm0,XMMWORD[rsi] - mov rbx,rdx + movdqu xmm0,XMMWORD[rsi] ; copy *(P256_POINT *)%rsi + mov rbx,rdx ; reassign movdqu xmm1,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm3,XMMWORD[48+rsi] movdqu xmm4,XMMWORD[64+rsi] movdqu xmm5,XMMWORD[80+rsi] - mov rdx,QWORD[((64+0))+rsi] + mov rdx,QWORD[((64+0))+rsi] ; load original in1_z mov r14,QWORD[((64+8))+rsi] mov r15,QWORD[((64+16))+rsi] mov r8,QWORD[((64+24))+rsi] @@ -4475,7 +4475,7 @@ movdqa XMMWORD[(384+16)+rsp],xmm5 por xmm5,xmm4 - movdqu xmm0,XMMWORD[rbx] + movdqu xmm0,XMMWORD[rbx] ; copy *(P256_POINT_AFFINE *)%rbx pshufd xmm3,xmm5,0xb1 movdqu xmm1,XMMWORD[16+rbx] movdqu xmm2,XMMWORD[32+rbx] @@ -4485,7 +4485,7 @@ pshufd xmm4,xmm5,0x1e movdqa XMMWORD[(416+16)+rsp],xmm1 por xmm1,xmm0 - movq xmm0,rdi + movq xmm0,rdi ; save %rdi movdqa XMMWORD[448+rsp],xmm2 movdqa XMMWORD[(448+16)+rsp],xmm3 por xmm3,xmm2 @@ -4493,33 +4493,33 @@ pxor xmm4,xmm4 por xmm3,xmm1 - lea rsi,[((64-128))+rsi] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montx + lea rsi,[((64-128))+rsi] ; %rsi is still valid + lea rdi,[32+rsp] ; Z1^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Z1sqr, in1_z); pcmpeqd xmm5,xmm4 pshufd xmm4,xmm3,0xb1 - mov rdx,QWORD[rbx] - - mov r9,r12 + mov rdx,QWORD[rbx] ; %rbx is still valid +; lea 0x00(%rbx), %rbx + mov r9,r12 ; harmonize sqr output and mul input por xmm4,xmm3 - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; in1infty pshufd xmm3,xmm4,0x1e mov r10,r13 por xmm4,xmm3 pxor xmm3,xmm3 mov r11,r14 pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 + pshufd xmm4,xmm4,0 ; in2infty lea rsi,[((32-128))+rsp] mov r12,r15 - lea rdi,[rsp] - call __ecp_nistz256_mul_montx + lea rdi,[rsp] ; U2 = X2*Z1^2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(U2, Z1sqr, in2_x); lea rbx,[320+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromx + lea rdi,[64+rsp] ; H = U2 - U1 + call __ecp_nistz256_sub_fromx ; p256_sub(H, U2, in1_x); mov rdx,QWORD[384+rsp] lea rbx,[384+rsp] @@ -4528,8 +4528,8 @@ lea rsi,[((-128+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[32+rsp] ; S2 = Z1^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(S2, Z1sqr, in1_z); mov rdx,QWORD[384+rsp] lea rbx,[384+rsp] @@ -4538,8 +4538,8 @@ lea rsi,[((-128+64))+rsp] mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] - lea rdi,[288+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[288+rsp] ; Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(res_z, H, in1_z); mov rdx,QWORD[448+rsp] lea rbx,[448+rsp] @@ -4548,28 +4548,28 @@ lea rsi,[((-128+32))+rsp] mov r11,QWORD[((16+32))+rsp] mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[32+rsp] ; S2 = Y2*Z1^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(S2, S2, in2_y); lea rbx,[352+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sub_fromx + lea rdi,[96+rsp] ; R = S2 - S1 + call __ecp_nistz256_sub_fromx ; p256_sub(R, S2, in1_y); mov rdx,QWORD[((0+64))+rsp] mov r14,QWORD[((8+64))+rsp] lea rsi,[((-128+64))+rsp] mov r15,QWORD[((16+64))+rsp] mov r8,QWORD[((24+64))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_sqr_montx + lea rdi,[128+rsp] ; H^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Hsqr, H); mov rdx,QWORD[((0+96))+rsp] mov r14,QWORD[((8+96))+rsp] lea rsi,[((-128+96))+rsp] mov r15,QWORD[((16+96))+rsp] mov r8,QWORD[((24+96))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_sqr_montx + lea rdi,[192+rsp] ; R^2 + call __ecp_nistz256_sqr_montx ; p256_sqr_mont(Rsqr, R); mov rdx,QWORD[128+rsp] lea rbx,[128+rsp] @@ -4578,8 +4578,8 @@ lea rsi,[((-128+64))+rsp] mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montx + lea rdi,[160+rsp] ; H^3 + call __ecp_nistz256_mul_montx ; p256_mul_mont(Hcub, Hsqr, H); mov rdx,QWORD[320+rsp] lea rbx,[320+rsp] @@ -4588,14 +4588,14 @@ lea rsi,[((-128+128))+rsp] mov r11,QWORD[((16+128))+rsp] mov r12,QWORD[((24+128))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_mul_montx - - - + lea rdi,[rsp] ; U1*H^2 + call __ecp_nistz256_mul_montx ; p256_mul_mont(U2, in1_x, Hsqr); +; lea 0(%rsp), %rsi +; lea 128(%rsp), %rdi # 2*U1*H^2 +; call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); xor r11,r11 - add r12,r12 + add r12,r12 ; a0:a3+a0:a3 lea rsi,[192+rsp] adc r13,r13 mov rax,r12 @@ -4621,11 +4621,11 @@ cmovc r9,r10 mov r10,QWORD[24+rsi] - call __ecp_nistz256_subx + call __ecp_nistz256_subx ; p256_sub(res_x, Rsqr, Hsqr); lea rbx,[160+rsp] lea rdi,[224+rsp] - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(res_x, res_x, Hcub); mov rax,QWORD[((0+0))+rsp] mov rbp,QWORD[((0+8))+rsp] @@ -4633,10 +4633,10 @@ mov r10,QWORD[((0+24))+rsp] lea rdi,[64+rsp] - call __ecp_nistz256_subx + call __ecp_nistz256_subx ; p256_sub(H, U2, res_x); - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 + mov QWORD[rdi],r12 ; save the result, as + mov QWORD[8+rdi],r13 ; __ecp_nistz256_sub doesn't mov QWORD[16+rdi],r8 mov QWORD[24+rdi],r9 mov rdx,QWORD[352+rsp] @@ -4647,7 +4647,7 @@ mov r11,QWORD[((16+160))+rsp] mov r12,QWORD[((24+160))+rsp] lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(S2, Hcub, in1_y); mov rdx,QWORD[96+rsp] lea rbx,[96+rsp] @@ -4657,15 +4657,15 @@ mov r11,QWORD[((16+64))+rsp] mov r12,QWORD[((24+64))+rsp] lea rdi,[64+rsp] - call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_montx ; p256_mul_mont(H, H, R); lea rbx,[32+rsp] lea rdi,[256+rsp] - call __ecp_nistz256_sub_fromx + call __ecp_nistz256_sub_fromx ; p256_sub(res_y, H, S2); - movq rdi,xmm0 + movq rdi,xmm0 ; restore %rdi - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_z, ONE, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[288+rsp] movdqa xmm2,xmm5 @@ -4676,7 +4676,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_z, in1_z, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -4689,7 +4689,7 @@ movdqu XMMWORD[64+rdi],xmm2 movdqu XMMWORD[80+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_x, in2_x, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[224+rsp] movdqa xmm2,xmm5 @@ -4700,7 +4700,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_x, in1_x, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -4713,7 +4713,7 @@ movdqu XMMWORD[rdi],xmm2 movdqu XMMWORD[16+rdi],xmm3 - movdqa xmm0,xmm5 + movdqa xmm0,xmm5 ; copy_conditional(res_y, in2_y, in1infty); movdqa xmm1,xmm5 pandn xmm0,XMMWORD[256+rsp] movdqa xmm2,xmm5 @@ -4724,7 +4724,7 @@ por xmm2,xmm0 por xmm3,xmm1 - movdqa xmm0,xmm4 + movdqa xmm0,xmm4 ; copy_conditional(res_y, in1_y, in2infty); movdqa xmm1,xmm4 pandn xmm0,xmm2 movdqa xmm2,xmm4 @@ -4775,30 +4775,30 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; end of prologue label + cmp rbx,r10 ; context->Rip<end of prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail lea rax,[16+rax] mov r12,QWORD[((-8))+rax] mov r13,QWORD[((-16))+rax] - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 jmp NEAR $L$common_seh_tail @@ -4817,25 +4817,25 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; end of prologue label + cmp rbx,r10 ; context->Rip<end of prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail - mov r10d,DWORD[8+r11] + mov r10d,DWORD[8+r11] ; HandlerData[2] lea rax,[r10*1+rax] mov rbp,QWORD[((-8))+rax] @@ -4844,40 +4844,40 @@ mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R15 $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -4967,106 +4967,106 @@ $L$SEH_info_ecp_nistz256_neg: DB 9,0,0,0 DD short_handler wrt ..imagebase - DD $L$neg_body wrt ..imagebase,$L$neg_epilogue wrt ..imagebase + DD $L$neg_body wrt ..imagebase,$L$neg_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_ecp_nistz256_ord_mul_mont_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$ord_mul_body wrt ..imagebase,$L$ord_mul_epilogue wrt ..imagebase + DD $L$ord_mul_body wrt ..imagebase,$L$ord_mul_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_ord_sqr_mont_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$ord_sqr_body wrt ..imagebase,$L$ord_sqr_epilogue wrt ..imagebase + DD $L$ord_sqr_body wrt ..imagebase,$L$ord_sqr_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_ord_mul_mont_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$ord_mulx_body wrt ..imagebase,$L$ord_mulx_epilogue wrt ..imagebase + DD $L$ord_mulx_body wrt ..imagebase,$L$ord_mulx_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_ord_sqr_mont_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$ord_sqrx_body wrt ..imagebase,$L$ord_sqrx_epilogue wrt ..imagebase + DD $L$ord_sqrx_body wrt ..imagebase,$L$ord_sqrx_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_mul_mont_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase + DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_sqr_mont_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$sqr_body wrt ..imagebase,$L$sqr_epilogue wrt ..imagebase + DD $L$sqr_body wrt ..imagebase,$L$sqr_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_mul_mont_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$mulx_body wrt ..imagebase,$L$mulx_epilogue wrt ..imagebase + DD $L$mulx_body wrt ..imagebase,$L$mulx_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_sqr_mont_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$sqrx_body wrt ..imagebase,$L$sqrx_epilogue wrt ..imagebase + DD $L$sqrx_body wrt ..imagebase,$L$sqrx_epilogue wrt ..imagebase ; HandlerData[] DD 48,0 $L$SEH_info_ecp_nistz256_select_wX_nohw: DB 0x01,0x33,0x16,0x00 - DB 0x33,0xf8,0x09,0x00 - DB 0x2e,0xe8,0x08,0x00 - DB 0x29,0xd8,0x07,0x00 - DB 0x24,0xc8,0x06,0x00 - DB 0x1f,0xb8,0x05,0x00 - DB 0x1a,0xa8,0x04,0x00 - DB 0x15,0x98,0x03,0x00 - DB 0x10,0x88,0x02,0x00 - DB 0x0c,0x78,0x01,0x00 - DB 0x08,0x68,0x00,0x00 - DB 0x04,0x01,0x15,0x00 + DB 0x33,0xf8,0x09,0x00 ; movaps 0x90(rsp),xmm15 + DB 0x2e,0xe8,0x08,0x00 ; movaps 0x80(rsp),xmm14 + DB 0x29,0xd8,0x07,0x00 ; movaps 0x70(rsp),xmm13 + DB 0x24,0xc8,0x06,0x00 ; movaps 0x60(rsp),xmm12 + DB 0x1f,0xb8,0x05,0x00 ; movaps 0x50(rsp),xmm11 + DB 0x1a,0xa8,0x04,0x00 ; movaps 0x40(rsp),xmm10 + DB 0x15,0x98,0x03,0x00 ; movaps 0x30(rsp),xmm9 + DB 0x10,0x88,0x02,0x00 ; movaps 0x20(rsp),xmm8 + DB 0x0c,0x78,0x01,0x00 ; movaps 0x10(rsp),xmm7 + DB 0x08,0x68,0x00,0x00 ; movaps 0x00(rsp),xmm6 + DB 0x04,0x01,0x15,0x00 ; sub rsp,0xa8 ALIGN 8 $L$SEH_info_ecp_nistz256_select_wX_avx2: DB 0x01,0x36,0x17,0x0b - DB 0x36,0xf8,0x09,0x00 - DB 0x31,0xe8,0x08,0x00 - DB 0x2c,0xd8,0x07,0x00 - DB 0x27,0xc8,0x06,0x00 - DB 0x22,0xb8,0x05,0x00 - DB 0x1d,0xa8,0x04,0x00 - DB 0x18,0x98,0x03,0x00 - DB 0x13,0x88,0x02,0x00 - DB 0x0e,0x78,0x01,0x00 - DB 0x09,0x68,0x00,0x00 - DB 0x04,0x01,0x15,0x00 - DB 0x00,0xb3,0x00,0x00 + DB 0x36,0xf8,0x09,0x00 ; vmovaps 0x90(rsp),xmm15 + DB 0x31,0xe8,0x08,0x00 ; vmovaps 0x80(rsp),xmm14 + DB 0x2c,0xd8,0x07,0x00 ; vmovaps 0x70(rsp),xmm13 + DB 0x27,0xc8,0x06,0x00 ; vmovaps 0x60(rsp),xmm12 + DB 0x22,0xb8,0x05,0x00 ; vmovaps 0x50(rsp),xmm11 + DB 0x1d,0xa8,0x04,0x00 ; vmovaps 0x40(rsp),xmm10 + DB 0x18,0x98,0x03,0x00 ; vmovaps 0x30(rsp),xmm9 + DB 0x13,0x88,0x02,0x00 ; vmovaps 0x20(rsp),xmm8 + DB 0x0e,0x78,0x01,0x00 ; vmovaps 0x10(rsp),xmm7 + DB 0x09,0x68,0x00,0x00 ; vmovaps 0x00(rsp),xmm6 + DB 0x04,0x01,0x15,0x00 ; sub rsp,0xa8 + DB 0x00,0xb3,0x00,0x00 ; set_frame r11 ALIGN 8 $L$SEH_info_ecp_nistz256_point_double_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$point_doubleq_body wrt ..imagebase,$L$point_doubleq_epilogue wrt ..imagebase + DD $L$point_doubleq_body wrt ..imagebase,$L$point_doubleq_epilogue wrt ..imagebase ; HandlerData[] DD 32*5+56,0 $L$SEH_info_ecp_nistz256_point_add_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$point_addq_body wrt ..imagebase,$L$point_addq_epilogue wrt ..imagebase + DD $L$point_addq_body wrt ..imagebase,$L$point_addq_epilogue wrt ..imagebase ; HandlerData[] DD 32*18+56,0 $L$SEH_info_ecp_nistz256_point_add_affine_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$add_affineq_body wrt ..imagebase,$L$add_affineq_epilogue wrt ..imagebase + DD $L$add_affineq_body wrt ..imagebase,$L$add_affineq_epilogue wrt ..imagebase ; HandlerData[] DD 32*15+56,0 ALIGN 8 $L$SEH_info_ecp_nistz256_point_double_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$point_doublex_body wrt ..imagebase,$L$point_doublex_epilogue wrt ..imagebase + DD $L$point_doublex_body wrt ..imagebase,$L$point_doublex_epilogue wrt ..imagebase ; HandlerData[] DD 32*5+56,0 $L$SEH_info_ecp_nistz256_point_add_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$point_addx_body wrt ..imagebase,$L$point_addx_epilogue wrt ..imagebase + DD $L$point_addx_body wrt ..imagebase,$L$point_addx_epilogue wrt ..imagebase ; HandlerData[] DD 32*18+56,0 $L$SEH_info_ecp_nistz256_point_add_affine_adx: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$add_affinex_body wrt ..imagebase,$L$add_affinex_epilogue wrt ..imagebase + DD $L$add_affinex_body wrt ..imagebase,$L$add_affinex_epilogue wrt ..imagebase ; HandlerData[] DD 32*15+56,0 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
diff --git a/gen/bcm/p256_beeu-x86_64-asm-apple.S b/gen/bcm/p256_beeu-x86_64-asm-apple.S index 35461b1..13bedb2 100644 --- a/gen/bcm/p256_beeu-x86_64-asm-apple.S +++ b/gen/bcm/p256_beeu-x86_64-asm-apple.S
@@ -32,7 +32,7 @@ movq %rdi,0(%rsp) - +// X=1, Y=0 movq $1,%r8 xorq %r9,%r9 xorq %r10,%r10 @@ -45,7 +45,7 @@ xorq %r15,%r15 xorq %rbp,%rbp - +// Copy a/n into B/A on the stack. vmovdqu 0(%rsi),%xmm0 vmovdqu 16(%rsi),%xmm1 vmovdqu %xmm0,48(%rsp) @@ -65,23 +65,23 @@ jz L$beeu_loop_end +// 0 < B < |n|, +// 0 < A <= |n|, +// (1) X*a == B (mod |n|), +// (2) (-1)*Y*a == A (mod |n|) - - - - - - - +// Now divide B by the maximum possible power of two in the +// integers, and divide X by the same value mod |n|. When we're +// done, (1) still holds. movq $1,%rcx - +// Note that B > 0 L$beeu_shift_loop_XB: movq %rcx,%rbx andq 48(%rsp),%rbx jnz L$beeu_shift_loop_end_XB - +// Ensure X is even and divide by two. movq $1,%rbx andq %r8,%rbx jz L$shift1_0 @@ -100,10 +100,10 @@ shlq $1,%rcx - - - - +// Test wraparound of the shift parameter. The probability to have 32 zeroes +// in a row is small Therefore having the value below equal $0x8000000 or +// $0x8000 does not affect the performance. We choose 0x8000000 because it +// is the maximal immediate value possible. cmpq $0x8000000,%rcx jne L$beeu_shift_loop_XB @@ -112,8 +112,8 @@ testq %rcx,%rcx jz L$beeu_no_shift_XB - - +// Copy shifted values. +// Remember not to override t3=rcx movq 8+48(%rsp),%rax movq 16+48(%rsp),%rbx movq 24+48(%rsp),%rsi @@ -127,16 +127,16 @@ L$beeu_no_shift_XB: - +// Same for A and Y. Afterwards, (2) still holds. movq $1,%rcx - +// Note that A > 0 L$beeu_shift_loop_YA: movq %rcx,%rbx andq 16(%rsp),%rbx jnz L$beeu_shift_loop_end_YA - +// Ensure X is even and divide by two. movq $1,%rbx andq %r12,%rbx jz L$shift1_1 @@ -155,10 +155,10 @@ shlq $1,%rcx - - - - +// Test wraparound of the shift parameter. The probability to have 32 zeroes +// in a row is small therefore having the value below equal $0x8000000 or +// $0x8000 Does not affect the performance. We choose 0x8000000 because it +// is the maximal immediate value possible. cmpq $0x8000000,%rcx jne L$beeu_shift_loop_YA @@ -167,8 +167,8 @@ testq %rcx,%rcx jz L$beeu_no_shift_YA - - +// Copy shifted values. +// Remember not to override t3=rcx movq 8+16(%rsp),%rax movq 16+16(%rsp),%rbx movq 24+16(%rsp),%rsi @@ -182,7 +182,7 @@ L$beeu_no_shift_YA: - +// T = B-A (A,B < 2^256) movq 48(%rsp),%rax movq 56(%rsp),%rbx movq 64(%rsp),%rsi @@ -190,10 +190,10 @@ subq 16(%rsp),%rax sbbq 24(%rsp),%rbx sbbq 32(%rsp),%rsi - sbbq 40(%rsp),%rcx + sbbq 40(%rsp),%rcx // borrow from shift jnc L$beeu_B_bigger_than_A - +// A = A - B movq 16(%rsp),%rax movq 24(%rsp),%rbx movq 32(%rsp),%rsi @@ -207,7 +207,7 @@ movq %rsi,32(%rsp) movq %rcx,40(%rsp) - +// Y = Y + X addq %r8,%r12 adcq %r9,%r13 adcq %r10,%r14 @@ -216,13 +216,13 @@ jmp L$beeu_loop L$beeu_B_bigger_than_A: - +// B = T = B - A movq %rax,48(%rsp) movq %rbx,56(%rsp) movq %rsi,64(%rsp) movq %rcx,72(%rsp) - +// X = Y + X addq %r12,%r8 adcq %r13,%r9 adcq %r14,%r10 @@ -232,21 +232,21 @@ jmp L$beeu_loop L$beeu_loop_end: +// The Euclid's algorithm loop ends when A == beeu(a,n); +// Therefore (-1)*Y*a == A (mod |n|), Y>0 - - - +// Verify that A = 1 ==> (-1)*Y*a = A = 1 (mod |n|) movq 16(%rsp),%rbx subq $1,%rbx orq 24(%rsp),%rbx orq 32(%rsp),%rbx orq 40(%rsp),%rbx - +// If not, fail. jnz L$beeu_err - - - +// From this point on, we no longer need X +// Therefore we use it as a temporary storage. +// X = n movq 0(%rdx),%r8 movq 8(%rdx),%r9 movq 16(%rdx),%r10 @@ -260,28 +260,28 @@ movq %r15,40(%rsp) movq %rbp,48(%rsp) - +// If Y>n ==> Y=Y-n subq %r8,%r12 sbbq %r9,%r13 sbbq %r10,%r14 sbbq %r11,%r15 sbbq $0,%rbp - +// Choose old Y or new Y cmovcq 16(%rsp),%r12 cmovcq 24(%rsp),%r13 cmovcq 32(%rsp),%r14 cmovcq 40(%rsp),%r15 jnc L$beeu_reduction_loop - +// X = n - Y (n, Y < 2^256), (Cancel the (-1)) subq %r12,%r8 sbbq %r13,%r9 sbbq %r14,%r10 sbbq %r15,%r11 L$beeu_save: - +// Save the inverse(<2^256) to out. movq 0(%rsp),%rdi movq %r8,0(%rdi) @@ -289,12 +289,12 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) - +// Return 1. movq $1,%rax jmp L$beeu_finish L$beeu_err: - +// Return 0. xorq %rax,%rax L$beeu_finish:
diff --git a/gen/bcm/p256_beeu-x86_64-asm-linux.S b/gen/bcm/p256_beeu-x86_64-asm-linux.S index 1056268..ee5a6f1 100644 --- a/gen/bcm/p256_beeu-x86_64-asm-linux.S +++ b/gen/bcm/p256_beeu-x86_64-asm-linux.S
@@ -39,7 +39,7 @@ .cfi_adjust_cfa_offset 80 movq %rdi,0(%rsp) - +// X=1, Y=0 movq $1,%r8 xorq %r9,%r9 xorq %r10,%r10 @@ -52,7 +52,7 @@ xorq %r15,%r15 xorq %rbp,%rbp - +// Copy a/n into B/A on the stack. vmovdqu 0(%rsi),%xmm0 vmovdqu 16(%rsi),%xmm1 vmovdqu %xmm0,48(%rsp) @@ -72,23 +72,23 @@ jz .Lbeeu_loop_end +// 0 < B < |n|, +// 0 < A <= |n|, +// (1) X*a == B (mod |n|), +// (2) (-1)*Y*a == A (mod |n|) - - - - - - - +// Now divide B by the maximum possible power of two in the +// integers, and divide X by the same value mod |n|. When we're +// done, (1) still holds. movq $1,%rcx - +// Note that B > 0 .Lbeeu_shift_loop_XB: movq %rcx,%rbx andq 48(%rsp),%rbx jnz .Lbeeu_shift_loop_end_XB - +// Ensure X is even and divide by two. movq $1,%rbx andq %r8,%rbx jz .Lshift1_0 @@ -107,10 +107,10 @@ shlq $1,%rcx - - - - +// Test wraparound of the shift parameter. The probability to have 32 zeroes +// in a row is small Therefore having the value below equal $0x8000000 or +// $0x8000 does not affect the performance. We choose 0x8000000 because it +// is the maximal immediate value possible. cmpq $0x8000000,%rcx jne .Lbeeu_shift_loop_XB @@ -119,8 +119,8 @@ testq %rcx,%rcx jz .Lbeeu_no_shift_XB - - +// Copy shifted values. +// Remember not to override t3=rcx movq 8+48(%rsp),%rax movq 16+48(%rsp),%rbx movq 24+48(%rsp),%rsi @@ -134,16 +134,16 @@ .Lbeeu_no_shift_XB: - +// Same for A and Y. Afterwards, (2) still holds. movq $1,%rcx - +// Note that A > 0 .Lbeeu_shift_loop_YA: movq %rcx,%rbx andq 16(%rsp),%rbx jnz .Lbeeu_shift_loop_end_YA - +// Ensure X is even and divide by two. movq $1,%rbx andq %r12,%rbx jz .Lshift1_1 @@ -162,10 +162,10 @@ shlq $1,%rcx - - - - +// Test wraparound of the shift parameter. The probability to have 32 zeroes +// in a row is small therefore having the value below equal $0x8000000 or +// $0x8000 Does not affect the performance. We choose 0x8000000 because it +// is the maximal immediate value possible. cmpq $0x8000000,%rcx jne .Lbeeu_shift_loop_YA @@ -174,8 +174,8 @@ testq %rcx,%rcx jz .Lbeeu_no_shift_YA - - +// Copy shifted values. +// Remember not to override t3=rcx movq 8+16(%rsp),%rax movq 16+16(%rsp),%rbx movq 24+16(%rsp),%rsi @@ -189,7 +189,7 @@ .Lbeeu_no_shift_YA: - +// T = B-A (A,B < 2^256) movq 48(%rsp),%rax movq 56(%rsp),%rbx movq 64(%rsp),%rsi @@ -197,10 +197,10 @@ subq 16(%rsp),%rax sbbq 24(%rsp),%rbx sbbq 32(%rsp),%rsi - sbbq 40(%rsp),%rcx + sbbq 40(%rsp),%rcx // borrow from shift jnc .Lbeeu_B_bigger_than_A - +// A = A - B movq 16(%rsp),%rax movq 24(%rsp),%rbx movq 32(%rsp),%rsi @@ -214,7 +214,7 @@ movq %rsi,32(%rsp) movq %rcx,40(%rsp) - +// Y = Y + X addq %r8,%r12 adcq %r9,%r13 adcq %r10,%r14 @@ -223,13 +223,13 @@ jmp .Lbeeu_loop .Lbeeu_B_bigger_than_A: - +// B = T = B - A movq %rax,48(%rsp) movq %rbx,56(%rsp) movq %rsi,64(%rsp) movq %rcx,72(%rsp) - +// X = Y + X addq %r12,%r8 adcq %r13,%r9 adcq %r14,%r10 @@ -239,21 +239,21 @@ jmp .Lbeeu_loop .Lbeeu_loop_end: +// The Euclid's algorithm loop ends when A == beeu(a,n); +// Therefore (-1)*Y*a == A (mod |n|), Y>0 - - - +// Verify that A = 1 ==> (-1)*Y*a = A = 1 (mod |n|) movq 16(%rsp),%rbx subq $1,%rbx orq 24(%rsp),%rbx orq 32(%rsp),%rbx orq 40(%rsp),%rbx - +// If not, fail. jnz .Lbeeu_err - - - +// From this point on, we no longer need X +// Therefore we use it as a temporary storage. +// X = n movq 0(%rdx),%r8 movq 8(%rdx),%r9 movq 16(%rdx),%r10 @@ -267,28 +267,28 @@ movq %r15,40(%rsp) movq %rbp,48(%rsp) - +// If Y>n ==> Y=Y-n subq %r8,%r12 sbbq %r9,%r13 sbbq %r10,%r14 sbbq %r11,%r15 sbbq $0,%rbp - +// Choose old Y or new Y cmovcq 16(%rsp),%r12 cmovcq 24(%rsp),%r13 cmovcq 32(%rsp),%r14 cmovcq 40(%rsp),%r15 jnc .Lbeeu_reduction_loop - +// X = n - Y (n, Y < 2^256), (Cancel the (-1)) subq %r12,%r8 sbbq %r13,%r9 sbbq %r14,%r10 sbbq %r15,%r11 .Lbeeu_save: - +// Save the inverse(<2^256) to out. movq 0(%rsp),%rdi movq %r8,0(%rdi) @@ -296,12 +296,12 @@ movq %r10,16(%rdi) movq %r11,24(%rdi) - +// Return 1. movq $1,%rax jmp .Lbeeu_finish .Lbeeu_err: - +// Return 0. xorq %rax,%rax .Lbeeu_finish:
diff --git a/gen/bcm/p256_beeu-x86_64-asm-win.asm b/gen/bcm/p256_beeu-x86_64-asm-win.asm index 5586c39..85bffc5 100644 --- a/gen/bcm/p256_beeu-x86_64-asm-win.asm +++ b/gen/bcm/p256_beeu-x86_64-asm-win.asm
@@ -51,7 +51,7 @@ mov QWORD[rsp],rdi - +; X=1, Y=0 mov r8,1 xor r9,r9 xor r10,r10 @@ -64,7 +64,7 @@ xor r15,r15 xor rbp,rbp - +; Copy a/n into B/A on the stack. vmovdqu xmm0,XMMWORD[rsi] vmovdqu xmm1,XMMWORD[16+rsi] vmovdqu XMMWORD[48+rsp],xmm0 @@ -84,23 +84,23 @@ jz NEAR $L$beeu_loop_end +; 0 < B < |n|, +; 0 < A <= |n|, +; (1) X*a == B (mod |n|), +; (2) (-1)*Y*a == A (mod |n|) - - - - - - - +; Now divide B by the maximum possible power of two in the +; integers, and divide X by the same value mod |n|. When we're +; done, (1) still holds. mov rcx,1 - +; Note that B > 0 $L$beeu_shift_loop_XB: mov rbx,rcx and rbx,QWORD[48+rsp] jnz NEAR $L$beeu_shift_loop_end_XB - +; Ensure X is even and divide by two. mov rbx,1 and rbx,r8 jz NEAR $L$shift1_0 @@ -119,10 +119,10 @@ shl rcx,1 - - - - +; Test wraparound of the shift parameter. The probability to have 32 zeroes +; in a row is small Therefore having the value below equal $0x8000000 or +; $0x8000 does not affect the performance. We choose 0x8000000 because it +; is the maximal immediate value possible. cmp rcx,0x8000000 jne NEAR $L$beeu_shift_loop_XB @@ -131,8 +131,8 @@ test rcx,rcx jz NEAR $L$beeu_no_shift_XB - - +; Copy shifted values. +; Remember not to override t3=rcx mov rax,QWORD[((8+48))+rsp] mov rbx,QWORD[((16+48))+rsp] mov rsi,QWORD[((24+48))+rsp] @@ -146,16 +146,16 @@ $L$beeu_no_shift_XB: - +; Same for A and Y. Afterwards, (2) still holds. mov rcx,1 - +; Note that A > 0 $L$beeu_shift_loop_YA: mov rbx,rcx and rbx,QWORD[16+rsp] jnz NEAR $L$beeu_shift_loop_end_YA - +; Ensure X is even and divide by two. mov rbx,1 and rbx,r12 jz NEAR $L$shift1_1 @@ -174,10 +174,10 @@ shl rcx,1 - - - - +; Test wraparound of the shift parameter. The probability to have 32 zeroes +; in a row is small therefore having the value below equal $0x8000000 or +; $0x8000 Does not affect the performance. We choose 0x8000000 because it +; is the maximal immediate value possible. cmp rcx,0x8000000 jne NEAR $L$beeu_shift_loop_YA @@ -186,8 +186,8 @@ test rcx,rcx jz NEAR $L$beeu_no_shift_YA - - +; Copy shifted values. +; Remember not to override t3=rcx mov rax,QWORD[((8+16))+rsp] mov rbx,QWORD[((16+16))+rsp] mov rsi,QWORD[((24+16))+rsp] @@ -201,7 +201,7 @@ $L$beeu_no_shift_YA: - +; T = B-A (A,B < 2^256) mov rax,QWORD[48+rsp] mov rbx,QWORD[56+rsp] mov rsi,QWORD[64+rsp] @@ -209,10 +209,10 @@ sub rax,QWORD[16+rsp] sbb rbx,QWORD[24+rsp] sbb rsi,QWORD[32+rsp] - sbb rcx,QWORD[40+rsp] + sbb rcx,QWORD[40+rsp] ; borrow from shift jnc NEAR $L$beeu_B_bigger_than_A - +; A = A - B mov rax,QWORD[16+rsp] mov rbx,QWORD[24+rsp] mov rsi,QWORD[32+rsp] @@ -226,7 +226,7 @@ mov QWORD[32+rsp],rsi mov QWORD[40+rsp],rcx - +; Y = Y + X add r12,r8 adc r13,r9 adc r14,r10 @@ -235,13 +235,13 @@ jmp NEAR $L$beeu_loop $L$beeu_B_bigger_than_A: - +; B = T = B - A mov QWORD[48+rsp],rax mov QWORD[56+rsp],rbx mov QWORD[64+rsp],rsi mov QWORD[72+rsp],rcx - +; X = Y + X add r8,r12 adc r9,r13 adc r10,r14 @@ -251,21 +251,21 @@ jmp NEAR $L$beeu_loop $L$beeu_loop_end: +; The Euclid's algorithm loop ends when A == beeu(a,n); +; Therefore (-1)*Y*a == A (mod |n|), Y>0 - - - +; Verify that A = 1 ==> (-1)*Y*a = A = 1 (mod |n|) mov rbx,QWORD[16+rsp] sub rbx,1 or rbx,QWORD[24+rsp] or rbx,QWORD[32+rsp] or rbx,QWORD[40+rsp] - +; If not, fail. jnz NEAR $L$beeu_err - - - +; From this point on, we no longer need X +; Therefore we use it as a temporary storage. +; X = n mov r8,QWORD[rdx] mov r9,QWORD[8+rdx] mov r10,QWORD[16+rdx] @@ -279,28 +279,28 @@ mov QWORD[40+rsp],r15 mov QWORD[48+rsp],rbp - +; If Y>n ==> Y=Y-n sub r12,r8 sbb r13,r9 sbb r14,r10 sbb r15,r11 sbb rbp,0 - +; Choose old Y or new Y cmovc r12,QWORD[16+rsp] cmovc r13,QWORD[24+rsp] cmovc r14,QWORD[32+rsp] cmovc r15,QWORD[40+rsp] jnc NEAR $L$beeu_reduction_loop - +; X = n - Y (n, Y < 2^256), (Cancel the (-1)) sub r8,r12 sbb r9,r13 sbb r10,r14 sbb r11,r15 $L$beeu_save: - +; Save the inverse(<2^256) to out. mov rdi,QWORD[rsp] mov QWORD[rdi],r8 @@ -308,12 +308,12 @@ mov QWORD[16+rdi],r10 mov QWORD[24+rdi],r11 - +; Return 1. mov rax,1 jmp NEAR $L$beeu_finish $L$beeu_err: - +; Return 0. xor rax,rax $L$beeu_finish:
diff --git a/gen/bcm/rdrand-x86_64-apple.S b/gen/bcm/rdrand-x86_64-apple.S index 1b76d7c..f995fd0 100644 --- a/gen/bcm/rdrand-x86_64-apple.S +++ b/gen/bcm/rdrand-x86_64-apple.S
@@ -6,9 +6,9 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) .text - - - +// CRYPTO_rdrand writes eight bytes of random data from the hardware RNG to +// |out|. It returns one on success or zero on hardware failure. +// int CRYPTO_rdrand(uint8_t out[8]); .globl _CRYPTO_rdrand .private_extern _CRYPTO_rdrand @@ -18,17 +18,17 @@ _CET_ENDBR xorq %rax,%rax rdrand %rdx - +// An add-with-carry of zero effectively sets %rax to the carry flag. adcq %rax,%rax movq %rdx,0(%rdi) ret - - - - +// CRYPTO_rdrand_multiple8_buf fills |len| bytes at |buf| with random data from +// the hardware RNG. The |len| argument must be a multiple of eight. It returns +// one on success and zero on hardware failure. +// int CRYPTO_rdrand_multiple8_buf(uint8_t *buf, size_t len); .globl _CRYPTO_rdrand_multiple8_buf .private_extern _CRYPTO_rdrand_multiple8_buf
diff --git a/gen/bcm/rdrand-x86_64-linux.S b/gen/bcm/rdrand-x86_64-linux.S index 52a1eb2..2859300 100644 --- a/gen/bcm/rdrand-x86_64-linux.S +++ b/gen/bcm/rdrand-x86_64-linux.S
@@ -6,9 +6,9 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) .text - - - +// CRYPTO_rdrand writes eight bytes of random data from the hardware RNG to +// |out|. It returns one on success or zero on hardware failure. +// int CRYPTO_rdrand(uint8_t out[8]); .globl CRYPTO_rdrand .hidden CRYPTO_rdrand .type CRYPTO_rdrand,@function @@ -18,17 +18,17 @@ _CET_ENDBR xorq %rax,%rax rdrand %rdx - +// An add-with-carry of zero effectively sets %rax to the carry flag. adcq %rax,%rax movq %rdx,0(%rdi) ret .cfi_endproc .size CRYPTO_rdrand,.-CRYPTO_rdrand - - - - +// CRYPTO_rdrand_multiple8_buf fills |len| bytes at |buf| with random data from +// the hardware RNG. The |len| argument must be a multiple of eight. It returns +// one on success and zero on hardware failure. +// int CRYPTO_rdrand_multiple8_buf(uint8_t *buf, size_t len); .globl CRYPTO_rdrand_multiple8_buf .hidden CRYPTO_rdrand_multiple8_buf .type CRYPTO_rdrand_multiple8_buf,@function
diff --git a/gen/bcm/rdrand-x86_64-win.asm b/gen/bcm/rdrand-x86_64-win.asm index 39d402f..d838065 100644 --- a/gen/bcm/rdrand-x86_64-win.asm +++ b/gen/bcm/rdrand-x86_64-win.asm
@@ -14,9 +14,9 @@ section .text code align=64 - - - +; CRYPTO_rdrand writes eight bytes of random data from the hardware RNG to +; |out|. It returns one on success or zero on hardware failure. +; int CRYPTO_rdrand(uint8_t out[8]); global CRYPTO_rdrand ALIGN 16 @@ -25,17 +25,17 @@ _CET_ENDBR xor rax,rax rdrand r8 - +; An add-with-carry of zero effectively sets %rax to the carry flag. adc rax,rax mov QWORD[rcx],r8 ret - - - - +; CRYPTO_rdrand_multiple8_buf fills |len| bytes at |buf| with random data from +; the hardware RNG. The |len| argument must be a multiple of eight. It returns +; one on success and zero on hardware failure. +; int CRYPTO_rdrand_multiple8_buf(uint8_t *buf, size_t len); global CRYPTO_rdrand_multiple8_buf ALIGN 16
diff --git a/gen/bcm/rsaz-avx2-apple.S b/gen/bcm/rsaz-avx2-apple.S index 803be09..0dac92a 100644 --- a/gen/bcm/rsaz-avx2-apple.S +++ b/gen/bcm/rsaz-avx2-apple.S
@@ -10,7 +10,7 @@ .private_extern _rsaz_1024_sqr_avx2 .p2align 6 -_rsaz_1024_sqr_avx2: +_rsaz_1024_sqr_avx2: // 702 cycles, 14% faster than rsaz_1024_mul_avx2 _CET_ENDBR leaq (%rsp),%rax @@ -30,23 +30,23 @@ vzeroupper movq %rax,%rbp - movq %rdx,%r13 + movq %rdx,%r13 // reassigned argument subq $832,%rsp movq %r13,%r15 - subq $-128,%rdi + subq $-128,%rdi // size optimization subq $-128,%rsi subq $-128,%r13 - andq $4095,%r15 + andq $4095,%r15 // see if %r13 crosses page addq $320,%r15 shrq $12,%r15 vpxor %ymm9,%ymm9,%ymm9 jz L$sqr_1024_no_n_copy - - - - +// unaligned 256-bit load that crosses page boundary can +// cause >2x performance degradation here, so if %r13 does +// cross page boundary, copy it to stack and make sure stack +// frame doesn't... subq $320,%rsp vmovdqu 0-128(%r13),%ymm0 andq $-2048,%rsp @@ -68,7 +68,7 @@ vmovdqu %ymm6,192-128(%r13) vmovdqu %ymm7,224-128(%r13) vmovdqu %ymm8,256-128(%r13) - vmovdqu %ymm9,288-128(%r13) + vmovdqu %ymm9,288-128(%r13) // %ymm9 is zero L$sqr_1024_no_n_copy: andq $-1024,%rsp @@ -82,18 +82,18 @@ vmovdqu 224-128(%rsi),%ymm7 vmovdqu 256-128(%rsi),%ymm8 - leaq 192(%rsp),%rbx + leaq 192(%rsp),%rbx // 64+128=192 vmovdqu L$and_mask(%rip),%ymm15 jmp L$OOP_GRANDE_SQR_1024 .p2align 5 L$OOP_GRANDE_SQR_1024: - leaq 576+128(%rsp),%r9 - leaq 448(%rsp),%r12 + leaq 576+128(%rsp),%r9 // size optimization + leaq 448(%rsp),%r12 // 64+128+256=448 - - - +// the squaring is performed as described in Variant B of +// "Speeding up Big-Number Squaring", so start by calculating +// the A*2=A+A vector vpaddq %ymm1,%ymm1,%ymm1 vpbroadcastq 0-128(%rsi),%ymm10 vpaddq %ymm2,%ymm2,%ymm2 @@ -115,7 +115,7 @@ vpmuludq 0-128(%rsi),%ymm10,%ymm0 vpbroadcastq 32-128(%rsi),%ymm11 - vmovdqu %ymm9,288-192(%rbx) + vmovdqu %ymm9,288-192(%rbx) // zero upper half vpmuludq %ymm10,%ymm1,%ymm1 vmovdqu %ymm9,320-448(%r12) vpmuludq %ymm10,%ymm2,%ymm2 @@ -252,10 +252,10 @@ vpmuludq 192-128(%rsi),%ymm10,%ymm12 vpaddq %ymm12,%ymm3,%ymm3 vpmuludq 192-128(%r9),%ymm10,%ymm14 - vpbroadcastq 256-128(%r15),%ymm0 + vpbroadcastq 256-128(%r15),%ymm0 // borrow %ymm0 for %ymm10 vpaddq %ymm14,%ymm4,%ymm4 vpmuludq 224-128(%r9),%ymm10,%ymm5 - vpbroadcastq 0+8-128(%r15),%ymm10 + vpbroadcastq 0+8-128(%r15),%ymm10 // for next iteration vpaddq 448-448(%r12),%ymm5,%ymm5 vmovdqu %ymm3,384-448(%r12) @@ -276,11 +276,11 @@ decl %r14d jnz L$OOP_SQR_1024 - - vmovdqu 256(%rsp),%ymm8 - vmovdqu 288(%rsp),%ymm1 - vmovdqu 320(%rsp),%ymm2 - leaq 192(%rsp),%rbx +// we need to fix indices 32-39 to avoid overflow + vmovdqu 256(%rsp),%ymm8 // 32*8-192(%rbx), + vmovdqu 288(%rsp),%ymm1 // 32*9-192(%rbx) + vmovdqu 320(%rsp),%ymm2 // 32*10-192(%rbx) + leaq 192(%rsp),%rbx // 64+128=192 vpsrlq $29,%ymm8,%ymm14 vpand %ymm15,%ymm8,%ymm8 @@ -376,17 +376,17 @@ vpaddq %ymm10,%ymm7,%ymm7 vpmuludq 256-128(%r13),%ymm12,%ymm14 vmovd %eax,%xmm12 - +// vmovdqu 32*1-8-128(%r13), %ymm11 # moved below vpaddq %ymm14,%ymm8,%ymm8 - +// vmovdqu 32*2-8-128(%r13), %ymm10 # moved below vpbroadcastq %xmm12,%ymm12 - vpmuludq 32-8-128(%r13),%ymm13,%ymm11 + vpmuludq 32-8-128(%r13),%ymm13,%ymm11 // see above vmovdqu 96-8-128(%r13),%ymm14 movq %rax,%rdx imulq -128(%r13),%rax vpaddq %ymm11,%ymm1,%ymm1 - vpmuludq 64-8-128(%r13),%ymm13,%ymm10 + vpmuludq 64-8-128(%r13),%ymm13,%ymm10 // see above vmovdqu 128-8-128(%r13),%ymm11 addq %rax,%r11 movq %rdx,%rax @@ -405,7 +405,7 @@ imull %ecx,%eax vpaddq %ymm11,%ymm4,%ymm4 vpmuludq %ymm13,%ymm10,%ymm10 -.byte 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 +.byte 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 // vmovdqu 32*7-8-128(%r13), %ymm11 andl $0x1fffffff,%eax vpaddq %ymm10,%ymm5,%ymm5 vpmuludq %ymm13,%ymm14,%ymm14 @@ -413,7 +413,7 @@ vpaddq %ymm14,%ymm6,%ymm6 vpmuludq %ymm13,%ymm11,%ymm11 vmovdqu 288-8-128(%r13),%ymm9 - vmovd %eax,%xmm0 + vmovd %eax,%xmm0 // borrow ACC0 for Y2 imulq -128(%r13),%rax vpaddq %ymm11,%ymm7,%ymm7 vpmuludq %ymm13,%ymm10,%ymm10 @@ -430,14 +430,14 @@ vpaddq %ymm14,%ymm1,%ymm1 vpmuludq %ymm0,%ymm13,%ymm13 vpmuludq %ymm12,%ymm11,%ymm11 -.byte 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff +.byte 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff // vmovdqu 32*4-16-128(%r13), %ymm14 vpaddq %ymm1,%ymm13,%ymm13 vpaddq %ymm11,%ymm2,%ymm2 vpmuludq %ymm12,%ymm10,%ymm10 vmovdqu 160-16-128(%r13),%ymm11 .byte 0x67 vmovq %xmm13,%rax - vmovdqu %ymm13,(%rsp) + vmovdqu %ymm13,(%rsp) // transfer %r9-%r12 vpaddq %ymm10,%ymm3,%ymm3 vpmuludq %ymm12,%ymm14,%ymm14 vmovdqu 192-16-128(%r13),%ymm10 @@ -454,7 +454,7 @@ addq %r12,%rax vpaddq %ymm14,%ymm7,%ymm7 vpmuludq %ymm12,%ymm11,%ymm11 - +// vmovdqu 32*2-24-128(%r13), %ymm14 # moved below movq %rax,%r9 imull %ecx,%eax vpaddq %ymm11,%ymm8,%ymm8 @@ -466,7 +466,7 @@ vpaddq %ymm10,%ymm9,%ymm9 vpbroadcastq %xmm12,%ymm12 - vpmuludq 64-24-128(%r13),%ymm0,%ymm14 + vpmuludq 64-24-128(%r13),%ymm0,%ymm14 // see above vmovdqu 128-24-128(%r13),%ymm10 movq %rax,%rdx imulq -128(%r13),%rax @@ -489,7 +489,7 @@ vpaddq %ymm10,%ymm4,%ymm3 vpmuludq %ymm0,%ymm14,%ymm14 vmovdqu 224-24-128(%r13),%ymm10 - imulq 24-128(%r13),%rdx + imulq 24-128(%r13),%rdx // future %r12 addq %rax,%r11 leaq (%r9,%r10,1),%rax vpaddq %ymm14,%ymm5,%ymm4 @@ -512,7 +512,7 @@ decl %r14d jnz L$OOP_REDUCE_1024 - leaq 448(%rsp),%r12 + leaq 448(%rsp),%r12 // size optimization vpaddq %ymm9,%ymm13,%ymm0 vpxor %ymm9,%ymm9,%ymm9 @@ -655,7 +655,7 @@ movq -8(%rax),%rbx - leaq (%rax),%rsp + leaq (%rax),%rsp // restore %rsp L$sqr_1024_epilogue: ret @@ -686,14 +686,14 @@ movq %rax,%rbp vzeroall - movq %rdx,%r13 + movq %rdx,%r13 // reassigned argument subq $64,%rsp - - - - - +// unaligned 256-bit load that crosses page boundary can +// cause severe performance degradation here, so if %rsi does +// cross page boundary, swap it with %r13 [meaning that caller +// is advised to lay down %rsi and %r13 next to each other, so +// that only one can cross page boundary]. .byte 0x67,0x67 movq %rsi,%r15 andq $4095,%r15 @@ -704,20 +704,20 @@ cmovnzq %r15,%r13 movq %rcx,%r15 - subq $-128,%rsi + subq $-128,%rsi // size optimization subq $-128,%rcx subq $-128,%rdi - andq $4095,%r15 + andq $4095,%r15 // see if %rcx crosses page addq $320,%r15 .byte 0x67,0x67 shrq $12,%r15 jz L$mul_1024_no_n_copy - - - - +// unaligned 256-bit load that crosses page boundary can +// cause severe performance degradation here, so if %rcx does +// cross page boundary, copy it to stack and make sure stack +// frame doesn't... subq $320,%rsp vmovdqu 0-128(%rcx),%ymm0 andq $-512,%rsp @@ -748,13 +748,13 @@ vpxor %ymm7,%ymm7,%ymm7 vmovdqu %ymm8,256-128(%rcx) vmovdqa %ymm0,%ymm8 - vmovdqu %ymm9,288-128(%rcx) + vmovdqu %ymm9,288-128(%rcx) // %ymm9 is zero after vzeroall L$mul_1024_no_n_copy: andq $-64,%rsp movq (%r13),%rbx vpbroadcastq (%r13),%ymm10 - vmovdqu %ymm0,(%rsp) + vmovdqu %ymm0,(%rsp) // clear top of stack xorq %r9,%r9 .byte 0x67 xorq %r10,%r10 @@ -763,12 +763,12 @@ vmovdqu L$and_mask(%rip),%ymm15 movl $9,%r14d - vmovdqu %ymm9,288-128(%rdi) + vmovdqu %ymm9,288-128(%rdi) // %ymm9 is zero after vzeroall jmp L$oop_mul_1024 .p2align 5 L$oop_mul_1024: - vpsrlq $29,%ymm3,%ymm9 + vpsrlq $29,%ymm3,%ymm9 // correct %ymm3(*) movq %rbx,%rax imulq -128(%rsi),%rax addq %r9,%rax @@ -794,7 +794,7 @@ vpbroadcastq %xmm11,%ymm11 vpaddq %ymm12,%ymm2,%ymm2 vpmuludq 96-128(%rsi),%ymm10,%ymm13 - vpand %ymm15,%ymm3,%ymm3 + vpand %ymm15,%ymm3,%ymm3 // correct %ymm3 vpaddq %ymm13,%ymm3,%ymm3 vpmuludq 128-128(%rsi),%ymm10,%ymm0 vpaddq %ymm0,%ymm4,%ymm4 @@ -803,7 +803,7 @@ vpmuludq 192-128(%rsi),%ymm10,%ymm13 vpaddq %ymm13,%ymm6,%ymm6 vpmuludq 224-128(%rsi),%ymm10,%ymm0 - vpermq $0x93,%ymm9,%ymm9 + vpermq $0x93,%ymm9,%ymm9 // correct %ymm3 vpaddq %ymm0,%ymm7,%ymm7 vpmuludq 256-128(%rsi),%ymm10,%ymm12 vpbroadcastq 8(%r13),%ymm10 @@ -837,10 +837,10 @@ vpmuludq 192-128(%rcx),%ymm11,%ymm12 vpaddq %ymm12,%ymm6,%ymm6 vpmuludq 224-128(%rcx),%ymm11,%ymm13 - vpblendd $3,%ymm14,%ymm9,%ymm12 + vpblendd $3,%ymm14,%ymm9,%ymm12 // correct %ymm3 vpaddq %ymm13,%ymm7,%ymm7 vpmuludq 256-128(%rcx),%ymm11,%ymm0 - vpaddq %ymm12,%ymm3,%ymm3 + vpaddq %ymm12,%ymm3,%ymm3 // correct %ymm3 vpaddq %ymm0,%ymm8,%ymm8 movq %rbx,%rax @@ -853,9 +853,9 @@ vmovdqu -8+64-128(%rsi),%ymm13 movq %r10,%rax - vpblendd $0xfc,%ymm14,%ymm9,%ymm9 + vpblendd $0xfc,%ymm14,%ymm9,%ymm9 // correct %ymm3 imull %r8d,%eax - vpaddq %ymm9,%ymm4,%ymm4 + vpaddq %ymm9,%ymm4,%ymm4 // correct %ymm3 andl $0x1fffffff,%eax imulq 16-128(%rsi),%rbx @@ -1043,7 +1043,7 @@ vpmuludq %ymm10,%ymm13,%ymm13 vpbroadcastq 32(%r13),%ymm10 vpaddq %ymm13,%ymm9,%ymm9 - addq $32,%r13 + addq $32,%r13 // %r13++ vmovdqu -24+32-128(%rcx),%ymm0 imulq -128(%rcx),%rax @@ -1054,9 +1054,9 @@ vpmuludq %ymm11,%ymm0,%ymm0 vmovq %xmm10,%rbx vmovdqu -24+96-128(%rcx),%ymm13 - vpaddq %ymm0,%ymm1,%ymm0 + vpaddq %ymm0,%ymm1,%ymm0 // %ymm0==%ymm0 vpmuludq %ymm11,%ymm12,%ymm12 - vmovdqu %ymm0,(%rsp) + vmovdqu %ymm0,(%rsp) // transfer %r9-%r12 vpaddq %ymm12,%ymm2,%ymm1 vmovdqu -24+128-128(%rcx),%ymm0 vpmuludq %ymm11,%ymm13,%ymm13 @@ -1213,7 +1213,7 @@ movq -8(%rax),%rbx - leaq (%rax),%rsp + leaq (%rax),%rsp // restore %rsp L$mul_1024_epilogue: ret @@ -1227,7 +1227,7 @@ _rsaz_1024_red2norm_avx2: _CET_ENDBR - subq $-128,%rsi + subq $-128,%rsi // size optimization xorq %rax,%rax movq -128(%rsi),%r8 movq -120(%rsi),%r9 @@ -1240,7 +1240,7 @@ addq %r8,%rax addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,0(%rdi) movq %r11,%rax movq -104(%rsi),%r8 @@ -1251,7 +1251,7 @@ shrq $12,%r10 addq %r8,%rax addq %r9,%rax - adcq $0,%r10 + adcq $0,%r10 // consume eventual carry movq %rax,8(%rdi) movq %r10,%rax movq -88(%rsi),%r11 @@ -1262,7 +1262,7 @@ shrq $18,%r9 addq %r11,%rax addq %r8,%rax - adcq $0,%r9 + adcq $0,%r9 // consume eventual carry movq %rax,16(%rdi) movq %r9,%rax movq -72(%rsi),%r10 @@ -1273,7 +1273,7 @@ shrq $24,%r8 addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,24(%rdi) movq %r8,%rax movq -56(%rsi),%r9 @@ -1287,7 +1287,7 @@ addq %r9,%rax addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,32(%rdi) movq %r8,%rax movq -32(%rsi),%r9 @@ -1298,7 +1298,7 @@ shrq $7,%r11 addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,40(%rdi) movq %r11,%rax movq -16(%rsi),%r8 @@ -1309,7 +1309,7 @@ shrq $13,%r10 addq %r8,%rax addq %r9,%rax - adcq $0,%r10 + adcq $0,%r10 // consume eventual carry movq %rax,48(%rdi) movq %r10,%rax movq 0(%rsi),%r11 @@ -1320,7 +1320,7 @@ shrq $19,%r9 addq %r11,%rax addq %r8,%rax - adcq $0,%r9 + adcq $0,%r9 // consume eventual carry movq %rax,56(%rdi) movq %r9,%rax movq 16(%rsi),%r10 @@ -1331,7 +1331,7 @@ shrq $25,%r8 addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,64(%rdi) movq %r8,%rax movq 32(%rsi),%r9 @@ -1345,7 +1345,7 @@ addq %r9,%rax addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,72(%rdi) movq %r8,%rax movq 56(%rsi),%r9 @@ -1356,7 +1356,7 @@ shrq $8,%r11 addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,80(%rdi) movq %r11,%rax movq 72(%rsi),%r8 @@ -1367,7 +1367,7 @@ shrq $14,%r10 addq %r8,%rax addq %r9,%rax - adcq $0,%r10 + adcq $0,%r10 // consume eventual carry movq %rax,88(%rdi) movq %r10,%rax movq 88(%rsi),%r11 @@ -1378,7 +1378,7 @@ shrq $20,%r9 addq %r11,%rax addq %r8,%rax - adcq $0,%r9 + adcq $0,%r9 // consume eventual carry movq %rax,96(%rdi) movq %r9,%rax movq 104(%rsi),%r10 @@ -1389,7 +1389,7 @@ shrq $26,%r8 addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,104(%rdi) movq %r8,%rax movq 120(%rsi),%r9 @@ -1403,7 +1403,7 @@ addq %r9,%rax addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,112(%rdi) movq %r8,%rax movq 144(%rsi),%r9 @@ -1414,7 +1414,7 @@ shrq $9,%r11 addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,120(%rdi) movq %r11,%rax ret @@ -1429,17 +1429,17 @@ _rsaz_1024_norm2red_avx2: _CET_ENDBR - subq $-128,%rdi + subq $-128,%rdi // size optimization movq (%rsi),%r8 movl $0x1fffffff,%eax movq 8(%rsi),%r9 movq %r8,%r11 shrq $0,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,-128(%rdi) movq %r8,%r10 shrq $29,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,-120(%rdi) shrdq $58,%r9,%r8 andq %rax,%r8 @@ -1447,7 +1447,7 @@ movq 16(%rsi),%r10 movq %r9,%r8 shrq $23,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,-104(%rdi) shrdq $52,%r10,%r9 andq %rax,%r9 @@ -1455,7 +1455,7 @@ movq 24(%rsi),%r11 movq %r10,%r9 shrq $17,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,-88(%rdi) shrdq $46,%r11,%r10 andq %rax,%r10 @@ -1463,7 +1463,7 @@ movq 32(%rsi),%r8 movq %r11,%r10 shrq $11,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,-72(%rdi) shrdq $40,%r8,%r11 andq %rax,%r11 @@ -1471,11 +1471,11 @@ movq 40(%rsi),%r9 movq %r8,%r11 shrq $5,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,-56(%rdi) movq %r8,%r10 shrq $34,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,-48(%rdi) shrdq $63,%r9,%r8 andq %rax,%r8 @@ -1483,7 +1483,7 @@ movq 48(%rsi),%r10 movq %r9,%r8 shrq $28,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,-32(%rdi) shrdq $57,%r10,%r9 andq %rax,%r9 @@ -1491,7 +1491,7 @@ movq 56(%rsi),%r11 movq %r10,%r9 shrq $22,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,-16(%rdi) shrdq $51,%r11,%r10 andq %rax,%r10 @@ -1499,7 +1499,7 @@ movq 64(%rsi),%r8 movq %r11,%r10 shrq $16,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,0(%rdi) shrdq $45,%r8,%r11 andq %rax,%r11 @@ -1507,7 +1507,7 @@ movq 72(%rsi),%r9 movq %r8,%r11 shrq $10,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,16(%rdi) shrdq $39,%r9,%r8 andq %rax,%r8 @@ -1515,11 +1515,11 @@ movq 80(%rsi),%r10 movq %r9,%r8 shrq $4,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,32(%rdi) movq %r9,%r11 shrq $33,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,40(%rdi) shrdq $62,%r10,%r9 andq %rax,%r9 @@ -1527,7 +1527,7 @@ movq 88(%rsi),%r11 movq %r10,%r9 shrq $27,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,56(%rdi) shrdq $56,%r11,%r10 andq %rax,%r10 @@ -1535,7 +1535,7 @@ movq 96(%rsi),%r8 movq %r11,%r10 shrq $21,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,72(%rdi) shrdq $50,%r8,%r11 andq %rax,%r11 @@ -1543,7 +1543,7 @@ movq 104(%rsi),%r9 movq %r8,%r11 shrq $15,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,88(%rdi) shrdq $44,%r9,%r8 andq %rax,%r8 @@ -1551,7 +1551,7 @@ movq 112(%rsi),%r10 movq %r9,%r8 shrq $9,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,104(%rdi) shrdq $38,%r10,%r9 andq %rax,%r9 @@ -1559,11 +1559,11 @@ movq 120(%rsi),%r11 movq %r10,%r9 shrq $3,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,120(%rdi) movq %r10,%r8 shrq $32,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,128(%rdi) shrdq $61,%r11,%r10 andq %rax,%r10 @@ -1571,12 +1571,12 @@ xorq %r8,%r8 movq %r11,%r10 shrq $26,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,144(%rdi) shrdq $55,%r8,%r11 andq %rax,%r11 movq %r11,152(%rdi) - movq %r8,160(%rdi) + movq %r8,160(%rdi) // zero movq %r8,168(%rdi) movq %r8,176(%rdi) movq %r8,184(%rdi) @@ -1627,7 +1627,7 @@ leaq -256(%rsp),%rsp andq $-32,%rsp leaq L$inc(%rip),%r10 - leaq -128(%rsp),%rax + leaq -128(%rsp),%rax // control u-op density vmovd %edx,%xmm4 vmovdqa (%r10),%ymm0 @@ -1674,7 +1674,7 @@ vpcmpeqd %ymm4,%ymm14,%ymm14 vpcmpeqd %ymm4,%ymm15,%ymm15 - vmovdqa -32(%r10),%ymm7 + vmovdqa -32(%r10),%ymm7 // .Lgather_permd leaq 128(%rsi),%rsi movl $9,%edx @@ -1720,7 +1720,7 @@ vpor %ymm3,%ymm5,%ymm5 vpor %ymm5,%ymm4,%ymm4 - vextracti128 $1,%ymm4,%xmm5 + vextracti128 $1,%ymm4,%xmm5 // upper half is cleared vpor %xmm4,%xmm5,%xmm5 vpermd %ymm5,%ymm7,%ymm5 vmovdqu %ymm5,(%rdi)
diff --git a/gen/bcm/rsaz-avx2-linux.S b/gen/bcm/rsaz-avx2-linux.S index 8374df4..dda873f 100644 --- a/gen/bcm/rsaz-avx2-linux.S +++ b/gen/bcm/rsaz-avx2-linux.S
@@ -10,7 +10,7 @@ .hidden rsaz_1024_sqr_avx2 .type rsaz_1024_sqr_avx2,@function .align 64 -rsaz_1024_sqr_avx2: +rsaz_1024_sqr_avx2: // 702 cycles, 14% faster than rsaz_1024_mul_avx2 .cfi_startproc _CET_ENDBR leaq (%rsp),%rax @@ -30,23 +30,23 @@ vzeroupper movq %rax,%rbp .cfi_def_cfa_register %rbp - movq %rdx,%r13 + movq %rdx,%r13 // reassigned argument subq $832,%rsp movq %r13,%r15 - subq $-128,%rdi + subq $-128,%rdi // size optimization subq $-128,%rsi subq $-128,%r13 - andq $4095,%r15 + andq $4095,%r15 // see if %r13 crosses page addq $320,%r15 shrq $12,%r15 vpxor %ymm9,%ymm9,%ymm9 jz .Lsqr_1024_no_n_copy - - - - +// unaligned 256-bit load that crosses page boundary can +// cause >2x performance degradation here, so if %r13 does +// cross page boundary, copy it to stack and make sure stack +// frame doesn't... subq $320,%rsp vmovdqu 0-128(%r13),%ymm0 andq $-2048,%rsp @@ -68,7 +68,7 @@ vmovdqu %ymm6,192-128(%r13) vmovdqu %ymm7,224-128(%r13) vmovdqu %ymm8,256-128(%r13) - vmovdqu %ymm9,288-128(%r13) + vmovdqu %ymm9,288-128(%r13) // %ymm9 is zero .Lsqr_1024_no_n_copy: andq $-1024,%rsp @@ -82,18 +82,18 @@ vmovdqu 224-128(%rsi),%ymm7 vmovdqu 256-128(%rsi),%ymm8 - leaq 192(%rsp),%rbx + leaq 192(%rsp),%rbx // 64+128=192 vmovdqu .Land_mask(%rip),%ymm15 jmp .LOOP_GRANDE_SQR_1024 .align 32 .LOOP_GRANDE_SQR_1024: - leaq 576+128(%rsp),%r9 - leaq 448(%rsp),%r12 + leaq 576+128(%rsp),%r9 // size optimization + leaq 448(%rsp),%r12 // 64+128+256=448 - - - +// the squaring is performed as described in Variant B of +// "Speeding up Big-Number Squaring", so start by calculating +// the A*2=A+A vector vpaddq %ymm1,%ymm1,%ymm1 vpbroadcastq 0-128(%rsi),%ymm10 vpaddq %ymm2,%ymm2,%ymm2 @@ -115,7 +115,7 @@ vpmuludq 0-128(%rsi),%ymm10,%ymm0 vpbroadcastq 32-128(%rsi),%ymm11 - vmovdqu %ymm9,288-192(%rbx) + vmovdqu %ymm9,288-192(%rbx) // zero upper half vpmuludq %ymm10,%ymm1,%ymm1 vmovdqu %ymm9,320-448(%r12) vpmuludq %ymm10,%ymm2,%ymm2 @@ -252,10 +252,10 @@ vpmuludq 192-128(%rsi),%ymm10,%ymm12 vpaddq %ymm12,%ymm3,%ymm3 vpmuludq 192-128(%r9),%ymm10,%ymm14 - vpbroadcastq 256-128(%r15),%ymm0 + vpbroadcastq 256-128(%r15),%ymm0 // borrow %ymm0 for %ymm10 vpaddq %ymm14,%ymm4,%ymm4 vpmuludq 224-128(%r9),%ymm10,%ymm5 - vpbroadcastq 0+8-128(%r15),%ymm10 + vpbroadcastq 0+8-128(%r15),%ymm10 // for next iteration vpaddq 448-448(%r12),%ymm5,%ymm5 vmovdqu %ymm3,384-448(%r12) @@ -276,11 +276,11 @@ decl %r14d jnz .LOOP_SQR_1024 - - vmovdqu 256(%rsp),%ymm8 - vmovdqu 288(%rsp),%ymm1 - vmovdqu 320(%rsp),%ymm2 - leaq 192(%rsp),%rbx +// we need to fix indices 32-39 to avoid overflow + vmovdqu 256(%rsp),%ymm8 // 32*8-192(%rbx), + vmovdqu 288(%rsp),%ymm1 // 32*9-192(%rbx) + vmovdqu 320(%rsp),%ymm2 // 32*10-192(%rbx) + leaq 192(%rsp),%rbx // 64+128=192 vpsrlq $29,%ymm8,%ymm14 vpand %ymm15,%ymm8,%ymm8 @@ -376,17 +376,17 @@ vpaddq %ymm10,%ymm7,%ymm7 vpmuludq 256-128(%r13),%ymm12,%ymm14 vmovd %eax,%xmm12 - +// vmovdqu 32*1-8-128(%r13), %ymm11 # moved below vpaddq %ymm14,%ymm8,%ymm8 - +// vmovdqu 32*2-8-128(%r13), %ymm10 # moved below vpbroadcastq %xmm12,%ymm12 - vpmuludq 32-8-128(%r13),%ymm13,%ymm11 + vpmuludq 32-8-128(%r13),%ymm13,%ymm11 // see above vmovdqu 96-8-128(%r13),%ymm14 movq %rax,%rdx imulq -128(%r13),%rax vpaddq %ymm11,%ymm1,%ymm1 - vpmuludq 64-8-128(%r13),%ymm13,%ymm10 + vpmuludq 64-8-128(%r13),%ymm13,%ymm10 // see above vmovdqu 128-8-128(%r13),%ymm11 addq %rax,%r11 movq %rdx,%rax @@ -405,7 +405,7 @@ imull %ecx,%eax vpaddq %ymm11,%ymm4,%ymm4 vpmuludq %ymm13,%ymm10,%ymm10 -.byte 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 +.byte 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 // vmovdqu 32*7-8-128(%r13), %ymm11 andl $0x1fffffff,%eax vpaddq %ymm10,%ymm5,%ymm5 vpmuludq %ymm13,%ymm14,%ymm14 @@ -413,7 +413,7 @@ vpaddq %ymm14,%ymm6,%ymm6 vpmuludq %ymm13,%ymm11,%ymm11 vmovdqu 288-8-128(%r13),%ymm9 - vmovd %eax,%xmm0 + vmovd %eax,%xmm0 // borrow ACC0 for Y2 imulq -128(%r13),%rax vpaddq %ymm11,%ymm7,%ymm7 vpmuludq %ymm13,%ymm10,%ymm10 @@ -430,14 +430,14 @@ vpaddq %ymm14,%ymm1,%ymm1 vpmuludq %ymm0,%ymm13,%ymm13 vpmuludq %ymm12,%ymm11,%ymm11 -.byte 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff +.byte 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff // vmovdqu 32*4-16-128(%r13), %ymm14 vpaddq %ymm1,%ymm13,%ymm13 vpaddq %ymm11,%ymm2,%ymm2 vpmuludq %ymm12,%ymm10,%ymm10 vmovdqu 160-16-128(%r13),%ymm11 .byte 0x67 vmovq %xmm13,%rax - vmovdqu %ymm13,(%rsp) + vmovdqu %ymm13,(%rsp) // transfer %r9-%r12 vpaddq %ymm10,%ymm3,%ymm3 vpmuludq %ymm12,%ymm14,%ymm14 vmovdqu 192-16-128(%r13),%ymm10 @@ -454,7 +454,7 @@ addq %r12,%rax vpaddq %ymm14,%ymm7,%ymm7 vpmuludq %ymm12,%ymm11,%ymm11 - +// vmovdqu 32*2-24-128(%r13), %ymm14 # moved below movq %rax,%r9 imull %ecx,%eax vpaddq %ymm11,%ymm8,%ymm8 @@ -466,7 +466,7 @@ vpaddq %ymm10,%ymm9,%ymm9 vpbroadcastq %xmm12,%ymm12 - vpmuludq 64-24-128(%r13),%ymm0,%ymm14 + vpmuludq 64-24-128(%r13),%ymm0,%ymm14 // see above vmovdqu 128-24-128(%r13),%ymm10 movq %rax,%rdx imulq -128(%r13),%rax @@ -489,7 +489,7 @@ vpaddq %ymm10,%ymm4,%ymm3 vpmuludq %ymm0,%ymm14,%ymm14 vmovdqu 224-24-128(%r13),%ymm10 - imulq 24-128(%r13),%rdx + imulq 24-128(%r13),%rdx // future %r12 addq %rax,%r11 leaq (%r9,%r10,1),%rax vpaddq %ymm14,%ymm5,%ymm4 @@ -512,7 +512,7 @@ decl %r14d jnz .LOOP_REDUCE_1024 - leaq 448(%rsp),%r12 + leaq 448(%rsp),%r12 // size optimization vpaddq %ymm9,%ymm13,%ymm0 vpxor %ymm9,%ymm9,%ymm9 @@ -655,7 +655,7 @@ .cfi_restore %rbp movq -8(%rax),%rbx .cfi_restore %rbx - leaq (%rax),%rsp + leaq (%rax),%rsp // restore %rsp .cfi_def_cfa_register %rsp .Lsqr_1024_epilogue: ret @@ -685,14 +685,14 @@ movq %rax,%rbp .cfi_def_cfa_register %rbp vzeroall - movq %rdx,%r13 + movq %rdx,%r13 // reassigned argument subq $64,%rsp - - - - - +// unaligned 256-bit load that crosses page boundary can +// cause severe performance degradation here, so if %rsi does +// cross page boundary, swap it with %r13 [meaning that caller +// is advised to lay down %rsi and %r13 next to each other, so +// that only one can cross page boundary]. .byte 0x67,0x67 movq %rsi,%r15 andq $4095,%r15 @@ -703,20 +703,20 @@ cmovnzq %r15,%r13 movq %rcx,%r15 - subq $-128,%rsi + subq $-128,%rsi // size optimization subq $-128,%rcx subq $-128,%rdi - andq $4095,%r15 + andq $4095,%r15 // see if %rcx crosses page addq $320,%r15 .byte 0x67,0x67 shrq $12,%r15 jz .Lmul_1024_no_n_copy - - - - +// unaligned 256-bit load that crosses page boundary can +// cause severe performance degradation here, so if %rcx does +// cross page boundary, copy it to stack and make sure stack +// frame doesn't... subq $320,%rsp vmovdqu 0-128(%rcx),%ymm0 andq $-512,%rsp @@ -747,13 +747,13 @@ vpxor %ymm7,%ymm7,%ymm7 vmovdqu %ymm8,256-128(%rcx) vmovdqa %ymm0,%ymm8 - vmovdqu %ymm9,288-128(%rcx) + vmovdqu %ymm9,288-128(%rcx) // %ymm9 is zero after vzeroall .Lmul_1024_no_n_copy: andq $-64,%rsp movq (%r13),%rbx vpbroadcastq (%r13),%ymm10 - vmovdqu %ymm0,(%rsp) + vmovdqu %ymm0,(%rsp) // clear top of stack xorq %r9,%r9 .byte 0x67 xorq %r10,%r10 @@ -762,12 +762,12 @@ vmovdqu .Land_mask(%rip),%ymm15 movl $9,%r14d - vmovdqu %ymm9,288-128(%rdi) + vmovdqu %ymm9,288-128(%rdi) // %ymm9 is zero after vzeroall jmp .Loop_mul_1024 .align 32 .Loop_mul_1024: - vpsrlq $29,%ymm3,%ymm9 + vpsrlq $29,%ymm3,%ymm9 // correct %ymm3(*) movq %rbx,%rax imulq -128(%rsi),%rax addq %r9,%rax @@ -793,7 +793,7 @@ vpbroadcastq %xmm11,%ymm11 vpaddq %ymm12,%ymm2,%ymm2 vpmuludq 96-128(%rsi),%ymm10,%ymm13 - vpand %ymm15,%ymm3,%ymm3 + vpand %ymm15,%ymm3,%ymm3 // correct %ymm3 vpaddq %ymm13,%ymm3,%ymm3 vpmuludq 128-128(%rsi),%ymm10,%ymm0 vpaddq %ymm0,%ymm4,%ymm4 @@ -802,7 +802,7 @@ vpmuludq 192-128(%rsi),%ymm10,%ymm13 vpaddq %ymm13,%ymm6,%ymm6 vpmuludq 224-128(%rsi),%ymm10,%ymm0 - vpermq $0x93,%ymm9,%ymm9 + vpermq $0x93,%ymm9,%ymm9 // correct %ymm3 vpaddq %ymm0,%ymm7,%ymm7 vpmuludq 256-128(%rsi),%ymm10,%ymm12 vpbroadcastq 8(%r13),%ymm10 @@ -836,10 +836,10 @@ vpmuludq 192-128(%rcx),%ymm11,%ymm12 vpaddq %ymm12,%ymm6,%ymm6 vpmuludq 224-128(%rcx),%ymm11,%ymm13 - vpblendd $3,%ymm14,%ymm9,%ymm12 + vpblendd $3,%ymm14,%ymm9,%ymm12 // correct %ymm3 vpaddq %ymm13,%ymm7,%ymm7 vpmuludq 256-128(%rcx),%ymm11,%ymm0 - vpaddq %ymm12,%ymm3,%ymm3 + vpaddq %ymm12,%ymm3,%ymm3 // correct %ymm3 vpaddq %ymm0,%ymm8,%ymm8 movq %rbx,%rax @@ -852,9 +852,9 @@ vmovdqu -8+64-128(%rsi),%ymm13 movq %r10,%rax - vpblendd $0xfc,%ymm14,%ymm9,%ymm9 + vpblendd $0xfc,%ymm14,%ymm9,%ymm9 // correct %ymm3 imull %r8d,%eax - vpaddq %ymm9,%ymm4,%ymm4 + vpaddq %ymm9,%ymm4,%ymm4 // correct %ymm3 andl $0x1fffffff,%eax imulq 16-128(%rsi),%rbx @@ -1042,7 +1042,7 @@ vpmuludq %ymm10,%ymm13,%ymm13 vpbroadcastq 32(%r13),%ymm10 vpaddq %ymm13,%ymm9,%ymm9 - addq $32,%r13 + addq $32,%r13 // %r13++ vmovdqu -24+32-128(%rcx),%ymm0 imulq -128(%rcx),%rax @@ -1053,9 +1053,9 @@ vpmuludq %ymm11,%ymm0,%ymm0 vmovq %xmm10,%rbx vmovdqu -24+96-128(%rcx),%ymm13 - vpaddq %ymm0,%ymm1,%ymm0 + vpaddq %ymm0,%ymm1,%ymm0 // %ymm0==%ymm0 vpmuludq %ymm11,%ymm12,%ymm12 - vmovdqu %ymm0,(%rsp) + vmovdqu %ymm0,(%rsp) // transfer %r9-%r12 vpaddq %ymm12,%ymm2,%ymm1 vmovdqu -24+128-128(%rcx),%ymm0 vpmuludq %ymm11,%ymm13,%ymm13 @@ -1212,7 +1212,7 @@ .cfi_restore %rbp movq -8(%rax),%rbx .cfi_restore %rbx - leaq (%rax),%rsp + leaq (%rax),%rsp // restore %rsp .cfi_def_cfa_register %rsp .Lmul_1024_epilogue: ret @@ -1225,7 +1225,7 @@ rsaz_1024_red2norm_avx2: .cfi_startproc _CET_ENDBR - subq $-128,%rsi + subq $-128,%rsi // size optimization xorq %rax,%rax movq -128(%rsi),%r8 movq -120(%rsi),%r9 @@ -1238,7 +1238,7 @@ addq %r8,%rax addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,0(%rdi) movq %r11,%rax movq -104(%rsi),%r8 @@ -1249,7 +1249,7 @@ shrq $12,%r10 addq %r8,%rax addq %r9,%rax - adcq $0,%r10 + adcq $0,%r10 // consume eventual carry movq %rax,8(%rdi) movq %r10,%rax movq -88(%rsi),%r11 @@ -1260,7 +1260,7 @@ shrq $18,%r9 addq %r11,%rax addq %r8,%rax - adcq $0,%r9 + adcq $0,%r9 // consume eventual carry movq %rax,16(%rdi) movq %r9,%rax movq -72(%rsi),%r10 @@ -1271,7 +1271,7 @@ shrq $24,%r8 addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,24(%rdi) movq %r8,%rax movq -56(%rsi),%r9 @@ -1285,7 +1285,7 @@ addq %r9,%rax addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,32(%rdi) movq %r8,%rax movq -32(%rsi),%r9 @@ -1296,7 +1296,7 @@ shrq $7,%r11 addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,40(%rdi) movq %r11,%rax movq -16(%rsi),%r8 @@ -1307,7 +1307,7 @@ shrq $13,%r10 addq %r8,%rax addq %r9,%rax - adcq $0,%r10 + adcq $0,%r10 // consume eventual carry movq %rax,48(%rdi) movq %r10,%rax movq 0(%rsi),%r11 @@ -1318,7 +1318,7 @@ shrq $19,%r9 addq %r11,%rax addq %r8,%rax - adcq $0,%r9 + adcq $0,%r9 // consume eventual carry movq %rax,56(%rdi) movq %r9,%rax movq 16(%rsi),%r10 @@ -1329,7 +1329,7 @@ shrq $25,%r8 addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,64(%rdi) movq %r8,%rax movq 32(%rsi),%r9 @@ -1343,7 +1343,7 @@ addq %r9,%rax addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,72(%rdi) movq %r8,%rax movq 56(%rsi),%r9 @@ -1354,7 +1354,7 @@ shrq $8,%r11 addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,80(%rdi) movq %r11,%rax movq 72(%rsi),%r8 @@ -1365,7 +1365,7 @@ shrq $14,%r10 addq %r8,%rax addq %r9,%rax - adcq $0,%r10 + adcq $0,%r10 // consume eventual carry movq %rax,88(%rdi) movq %r10,%rax movq 88(%rsi),%r11 @@ -1376,7 +1376,7 @@ shrq $20,%r9 addq %r11,%rax addq %r8,%rax - adcq $0,%r9 + adcq $0,%r9 // consume eventual carry movq %rax,96(%rdi) movq %r9,%rax movq 104(%rsi),%r10 @@ -1387,7 +1387,7 @@ shrq $26,%r8 addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,104(%rdi) movq %r8,%rax movq 120(%rsi),%r9 @@ -1401,7 +1401,7 @@ addq %r9,%rax addq %r10,%rax addq %r11,%rax - adcq $0,%r8 + adcq $0,%r8 // consume eventual carry movq %rax,112(%rdi) movq %r8,%rax movq 144(%rsi),%r9 @@ -1412,7 +1412,7 @@ shrq $9,%r11 addq %r9,%rax addq %r10,%rax - adcq $0,%r11 + adcq $0,%r11 // consume eventual carry movq %rax,120(%rdi) movq %r11,%rax ret @@ -1426,17 +1426,17 @@ rsaz_1024_norm2red_avx2: .cfi_startproc _CET_ENDBR - subq $-128,%rdi + subq $-128,%rdi // size optimization movq (%rsi),%r8 movl $0x1fffffff,%eax movq 8(%rsi),%r9 movq %r8,%r11 shrq $0,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,-128(%rdi) movq %r8,%r10 shrq $29,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,-120(%rdi) shrdq $58,%r9,%r8 andq %rax,%r8 @@ -1444,7 +1444,7 @@ movq 16(%rsi),%r10 movq %r9,%r8 shrq $23,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,-104(%rdi) shrdq $52,%r10,%r9 andq %rax,%r9 @@ -1452,7 +1452,7 @@ movq 24(%rsi),%r11 movq %r10,%r9 shrq $17,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,-88(%rdi) shrdq $46,%r11,%r10 andq %rax,%r10 @@ -1460,7 +1460,7 @@ movq 32(%rsi),%r8 movq %r11,%r10 shrq $11,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,-72(%rdi) shrdq $40,%r8,%r11 andq %rax,%r11 @@ -1468,11 +1468,11 @@ movq 40(%rsi),%r9 movq %r8,%r11 shrq $5,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,-56(%rdi) movq %r8,%r10 shrq $34,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,-48(%rdi) shrdq $63,%r9,%r8 andq %rax,%r8 @@ -1480,7 +1480,7 @@ movq 48(%rsi),%r10 movq %r9,%r8 shrq $28,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,-32(%rdi) shrdq $57,%r10,%r9 andq %rax,%r9 @@ -1488,7 +1488,7 @@ movq 56(%rsi),%r11 movq %r10,%r9 shrq $22,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,-16(%rdi) shrdq $51,%r11,%r10 andq %rax,%r10 @@ -1496,7 +1496,7 @@ movq 64(%rsi),%r8 movq %r11,%r10 shrq $16,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,0(%rdi) shrdq $45,%r8,%r11 andq %rax,%r11 @@ -1504,7 +1504,7 @@ movq 72(%rsi),%r9 movq %r8,%r11 shrq $10,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,16(%rdi) shrdq $39,%r9,%r8 andq %rax,%r8 @@ -1512,11 +1512,11 @@ movq 80(%rsi),%r10 movq %r9,%r8 shrq $4,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,32(%rdi) movq %r9,%r11 shrq $33,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,40(%rdi) shrdq $62,%r10,%r9 andq %rax,%r9 @@ -1524,7 +1524,7 @@ movq 88(%rsi),%r11 movq %r10,%r9 shrq $27,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,56(%rdi) shrdq $56,%r11,%r10 andq %rax,%r10 @@ -1532,7 +1532,7 @@ movq 96(%rsi),%r8 movq %r11,%r10 shrq $21,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,72(%rdi) shrdq $50,%r8,%r11 andq %rax,%r11 @@ -1540,7 +1540,7 @@ movq 104(%rsi),%r9 movq %r8,%r11 shrq $15,%r11 - andq %rax,%r11 + andq %rax,%r11 // &0x1fffffff movq %r11,88(%rdi) shrdq $44,%r9,%r8 andq %rax,%r8 @@ -1548,7 +1548,7 @@ movq 112(%rsi),%r10 movq %r9,%r8 shrq $9,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,104(%rdi) shrdq $38,%r10,%r9 andq %rax,%r9 @@ -1556,11 +1556,11 @@ movq 120(%rsi),%r11 movq %r10,%r9 shrq $3,%r9 - andq %rax,%r9 + andq %rax,%r9 // &0x1fffffff movq %r9,120(%rdi) movq %r10,%r8 shrq $32,%r8 - andq %rax,%r8 + andq %rax,%r8 // &0x1fffffff movq %r8,128(%rdi) shrdq $61,%r11,%r10 andq %rax,%r10 @@ -1568,12 +1568,12 @@ xorq %r8,%r8 movq %r11,%r10 shrq $26,%r10 - andq %rax,%r10 + andq %rax,%r10 // &0x1fffffff movq %r10,144(%rdi) shrdq $55,%r8,%r11 andq %rax,%r11 movq %r11,152(%rdi) - movq %r8,160(%rdi) + movq %r8,160(%rdi) // zero movq %r8,168(%rdi) movq %r8,176(%rdi) movq %r8,184(%rdi) @@ -1622,7 +1622,7 @@ leaq -256(%rsp),%rsp andq $-32,%rsp leaq .Linc(%rip),%r10 - leaq -128(%rsp),%rax + leaq -128(%rsp),%rax // control u-op density vmovd %edx,%xmm4 vmovdqa (%r10),%ymm0 @@ -1669,7 +1669,7 @@ vpcmpeqd %ymm4,%ymm14,%ymm14 vpcmpeqd %ymm4,%ymm15,%ymm15 - vmovdqa -32(%r10),%ymm7 + vmovdqa -32(%r10),%ymm7 // .Lgather_permd leaq 128(%rsi),%rsi movl $9,%edx @@ -1715,7 +1715,7 @@ vpor %ymm3,%ymm5,%ymm5 vpor %ymm5,%ymm4,%ymm4 - vextracti128 $1,%ymm4,%xmm5 + vextracti128 $1,%ymm4,%xmm5 // upper half is cleared vpor %xmm4,%xmm5,%xmm5 vpermd %ymm5,%ymm7,%ymm5 vmovdqu %ymm5,(%rdi)
diff --git a/gen/bcm/rsaz-avx2-win.asm b/gen/bcm/rsaz-avx2-win.asm index 0dfaee6..a142c27 100644 --- a/gen/bcm/rsaz-avx2-win.asm +++ b/gen/bcm/rsaz-avx2-win.asm
@@ -28,7 +28,7 @@ mov rcx,r9 mov r8,QWORD[40+rsp] - +; 702 cycles, 14% faster than rsaz_1024_mul_avx2 _CET_ENDBR lea rax,[rsp] @@ -60,23 +60,23 @@ $L$sqr_1024_body: mov rbp,rax - mov r13,rdx + mov r13,rdx ; reassigned argument sub rsp,832 mov r15,r13 - sub rdi,-128 + sub rdi,-128 ; size optimization sub rsi,-128 sub r13,-128 - and r15,4095 + and r15,4095 ; see if %r13 crosses page add r15,32*10 shr r15,12 vpxor ymm9,ymm9,ymm9 jz NEAR $L$sqr_1024_no_n_copy - - - - +; unaligned 256-bit load that crosses page boundary can +; cause >2x performance degradation here, so if %r13 does +; cross page boundary, copy it to stack and make sure stack +; frame doesn't... sub rsp,32*10 vmovdqu ymm0,YMMWORD[((0-128))+r13] and rsp,-2048 @@ -98,7 +98,7 @@ vmovdqu YMMWORD[(192-128)+r13],ymm6 vmovdqu YMMWORD[(224-128)+r13],ymm7 vmovdqu YMMWORD[(256-128)+r13],ymm8 - vmovdqu YMMWORD[(288-128)+r13],ymm9 + vmovdqu YMMWORD[(288-128)+r13],ymm9 ; %ymm9 is zero $L$sqr_1024_no_n_copy: and rsp,-1024 @@ -112,18 +112,18 @@ vmovdqu ymm7,YMMWORD[((224-128))+rsi] vmovdqu ymm8,YMMWORD[((256-128))+rsi] - lea rbx,[192+rsp] + lea rbx,[192+rsp] ; 64+128=192 vmovdqu ymm15,YMMWORD[$L$and_mask] jmp NEAR $L$OOP_GRANDE_SQR_1024 ALIGN 32 $L$OOP_GRANDE_SQR_1024: - lea r9,[((576+128))+rsp] - lea r12,[448+rsp] + lea r9,[((576+128))+rsp] ; size optimization + lea r12,[448+rsp] ; 64+128+256=448 - - - +; the squaring is performed as described in Variant B of +; "Speeding up Big-Number Squaring", so start by calculating +; the A*2=A+A vector vpaddq ymm1,ymm1,ymm1 vpbroadcastq ymm10,QWORD[((0-128))+rsi] vpaddq ymm2,ymm2,ymm2 @@ -145,7 +145,7 @@ vpmuludq ymm0,ymm10,YMMWORD[((0-128))+rsi] vpbroadcastq ymm11,QWORD[((32-128))+rsi] - vmovdqu YMMWORD[(288-192)+rbx],ymm9 + vmovdqu YMMWORD[(288-192)+rbx],ymm9 ; zero upper half vpmuludq ymm1,ymm1,ymm10 vmovdqu YMMWORD[(320-448)+r12],ymm9 vpmuludq ymm2,ymm2,ymm10 @@ -282,10 +282,10 @@ vpmuludq ymm12,ymm10,YMMWORD[((192-128))+rsi] vpaddq ymm3,ymm3,ymm12 vpmuludq ymm14,ymm10,YMMWORD[((192-128))+r9] - vpbroadcastq ymm0,QWORD[((256-128))+r15] + vpbroadcastq ymm0,QWORD[((256-128))+r15] ; borrow %ymm0 for %ymm10 vpaddq ymm4,ymm4,ymm14 vpmuludq ymm5,ymm10,YMMWORD[((224-128))+r9] - vpbroadcastq ymm10,QWORD[((0+8-128))+r15] + vpbroadcastq ymm10,QWORD[((0+8-128))+r15] ; for next iteration vpaddq ymm5,ymm5,YMMWORD[((448-448))+r12] vmovdqu YMMWORD[(384-448)+r12],ymm3 @@ -306,11 +306,11 @@ dec r14d jnz NEAR $L$OOP_SQR_1024 - - vmovdqu ymm8,YMMWORD[256+rsp] - vmovdqu ymm1,YMMWORD[288+rsp] - vmovdqu ymm2,YMMWORD[320+rsp] - lea rbx,[192+rsp] +; we need to fix indices 32-39 to avoid overflow + vmovdqu ymm8,YMMWORD[256+rsp] ; 32*8-192(%rbx), + vmovdqu ymm1,YMMWORD[288+rsp] ; 32*9-192(%rbx) + vmovdqu ymm2,YMMWORD[320+rsp] ; 32*10-192(%rbx) + lea rbx,[192+rsp] ; 64+128=192 vpsrlq ymm14,ymm8,29 vpand ymm8,ymm8,ymm15 @@ -406,17 +406,17 @@ vpaddq ymm7,ymm7,ymm10 vpmuludq ymm14,ymm12,YMMWORD[((256-128))+r13] vmovd xmm12,eax - +; vmovdqu 32*1-8-128(%r13), %ymm11 # moved below vpaddq ymm8,ymm8,ymm14 - +; vmovdqu 32*2-8-128(%r13), %ymm10 # moved below vpbroadcastq ymm12,xmm12 - vpmuludq ymm11,ymm13,YMMWORD[((32-8-128))+r13] + vpmuludq ymm11,ymm13,YMMWORD[((32-8-128))+r13] ; see above vmovdqu ymm14,YMMWORD[((96-8-128))+r13] mov rdx,rax imul rax,QWORD[((-128))+r13] vpaddq ymm1,ymm1,ymm11 - vpmuludq ymm10,ymm13,YMMWORD[((64-8-128))+r13] + vpmuludq ymm10,ymm13,YMMWORD[((64-8-128))+r13] ; see above vmovdqu ymm11,YMMWORD[((128-8-128))+r13] add r11,rax mov rax,rdx @@ -435,7 +435,7 @@ imul eax,ecx vpaddq ymm4,ymm4,ymm11 vpmuludq ymm10,ymm10,ymm13 - DB 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 + DB 0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00 ; vmovdqu 32*7-8-128(%r13), %ymm11 and eax,0x1fffffff vpaddq ymm5,ymm5,ymm10 vpmuludq ymm14,ymm14,ymm13 @@ -443,7 +443,7 @@ vpaddq ymm6,ymm6,ymm14 vpmuludq ymm11,ymm11,ymm13 vmovdqu ymm9,YMMWORD[((288-8-128))+r13] - vmovd xmm0,eax + vmovd xmm0,eax ; borrow ACC0 for Y2 imul rax,QWORD[((-128))+r13] vpaddq ymm7,ymm7,ymm11 vpmuludq ymm10,ymm10,ymm13 @@ -460,14 +460,14 @@ vpaddq ymm1,ymm1,ymm14 vpmuludq ymm13,ymm13,ymm0 vpmuludq ymm11,ymm11,ymm12 - DB 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff + DB 0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff ; vmovdqu 32*4-16-128(%r13), %ymm14 vpaddq ymm13,ymm13,ymm1 vpaddq ymm2,ymm2,ymm11 vpmuludq ymm10,ymm10,ymm12 vmovdqu ymm11,YMMWORD[((160-16-128))+r13] DB 0x67 vmovq rax,xmm13 - vmovdqu YMMWORD[rsp],ymm13 + vmovdqu YMMWORD[rsp],ymm13 ; transfer %r9-%r12 vpaddq ymm3,ymm3,ymm10 vpmuludq ymm14,ymm14,ymm12 vmovdqu ymm10,YMMWORD[((192-16-128))+r13] @@ -484,7 +484,7 @@ add rax,r12 vpaddq ymm7,ymm7,ymm14 vpmuludq ymm11,ymm11,ymm12 - +; vmovdqu 32*2-24-128(%r13), %ymm14 # moved below mov r9,rax imul eax,ecx vpaddq ymm8,ymm8,ymm11 @@ -496,7 +496,7 @@ vpaddq ymm9,ymm9,ymm10 vpbroadcastq ymm12,xmm12 - vpmuludq ymm14,ymm0,YMMWORD[((64-24-128))+r13] + vpmuludq ymm14,ymm0,YMMWORD[((64-24-128))+r13] ; see above vmovdqu ymm10,YMMWORD[((128-24-128))+r13] mov rdx,rax imul rax,QWORD[((-128))+r13] @@ -519,7 +519,7 @@ vpaddq ymm3,ymm4,ymm10 vpmuludq ymm14,ymm14,ymm0 vmovdqu ymm10,YMMWORD[((224-24-128))+r13] - imul rdx,QWORD[((24-128))+r13] + imul rdx,QWORD[((24-128))+r13] ; future %r12 add r11,rax lea rax,[r10*1+r9] vpaddq ymm4,ymm5,ymm14 @@ -542,7 +542,7 @@ dec r14d jnz NEAR $L$OOP_REDUCE_1024 - lea r12,[448+rsp] + lea r12,[448+rsp] ; size optimization vpaddq ymm0,ymm13,ymm9 vpxor ymm9,ymm9,ymm9 @@ -696,7 +696,7 @@ mov rbx,QWORD[((-8))+rax] - lea rsp,[rax] + lea rsp,[rax] ; restore %rsp $L$sqr_1024_epilogue: mov rdi,QWORD[8+rsp] ;WIN64 epilogue @@ -751,14 +751,14 @@ mov rbp,rax vzeroall - mov r13,rdx + mov r13,rdx ; reassigned argument sub rsp,64 - - - - - +; unaligned 256-bit load that crosses page boundary can +; cause severe performance degradation here, so if %rsi does +; cross page boundary, swap it with %r13 [meaning that caller +; is advised to lay down %rsi and %r13 next to each other, so +; that only one can cross page boundary]. DB 0x67,0x67 mov r15,rsi and r15,4095 @@ -769,20 +769,20 @@ cmovnz r13,r15 mov r15,rcx - sub rsi,-128 + sub rsi,-128 ; size optimization sub rcx,-128 sub rdi,-128 - and r15,4095 + and r15,4095 ; see if %rcx crosses page add r15,32*10 DB 0x67,0x67 shr r15,12 jz NEAR $L$mul_1024_no_n_copy - - - - +; unaligned 256-bit load that crosses page boundary can +; cause severe performance degradation here, so if %rcx does +; cross page boundary, copy it to stack and make sure stack +; frame doesn't... sub rsp,32*10 vmovdqu ymm0,YMMWORD[((0-128))+rcx] and rsp,-512 @@ -813,13 +813,13 @@ vpxor ymm7,ymm7,ymm7 vmovdqu YMMWORD[(256-128)+rcx],ymm8 vmovdqa ymm8,ymm0 - vmovdqu YMMWORD[(288-128)+rcx],ymm9 + vmovdqu YMMWORD[(288-128)+rcx],ymm9 ; %ymm9 is zero after vzeroall $L$mul_1024_no_n_copy: and rsp,-64 mov rbx,QWORD[r13] vpbroadcastq ymm10,QWORD[r13] - vmovdqu YMMWORD[rsp],ymm0 + vmovdqu YMMWORD[rsp],ymm0 ; clear top of stack xor r9,r9 DB 0x67 xor r10,r10 @@ -828,12 +828,12 @@ vmovdqu ymm15,YMMWORD[$L$and_mask] mov r14d,9 - vmovdqu YMMWORD[(288-128)+rdi],ymm9 + vmovdqu YMMWORD[(288-128)+rdi],ymm9 ; %ymm9 is zero after vzeroall jmp NEAR $L$oop_mul_1024 ALIGN 32 $L$oop_mul_1024: - vpsrlq ymm9,ymm3,29 + vpsrlq ymm9,ymm3,29 ; correct %ymm3(*) mov rax,rbx imul rax,QWORD[((-128))+rsi] add rax,r9 @@ -859,7 +859,7 @@ vpbroadcastq ymm11,xmm11 vpaddq ymm2,ymm2,ymm12 vpmuludq ymm13,ymm10,YMMWORD[((96-128))+rsi] - vpand ymm3,ymm3,ymm15 + vpand ymm3,ymm3,ymm15 ; correct %ymm3 vpaddq ymm3,ymm3,ymm13 vpmuludq ymm0,ymm10,YMMWORD[((128-128))+rsi] vpaddq ymm4,ymm4,ymm0 @@ -868,7 +868,7 @@ vpmuludq ymm13,ymm10,YMMWORD[((192-128))+rsi] vpaddq ymm6,ymm6,ymm13 vpmuludq ymm0,ymm10,YMMWORD[((224-128))+rsi] - vpermq ymm9,ymm9,0x93 + vpermq ymm9,ymm9,0x93 ; correct %ymm3 vpaddq ymm7,ymm7,ymm0 vpmuludq ymm12,ymm10,YMMWORD[((256-128))+rsi] vpbroadcastq ymm10,QWORD[8+r13] @@ -902,10 +902,10 @@ vpmuludq ymm12,ymm11,YMMWORD[((192-128))+rcx] vpaddq ymm6,ymm6,ymm12 vpmuludq ymm13,ymm11,YMMWORD[((224-128))+rcx] - vpblendd ymm12,ymm9,ymm14,3 + vpblendd ymm12,ymm9,ymm14,3 ; correct %ymm3 vpaddq ymm7,ymm7,ymm13 vpmuludq ymm0,ymm11,YMMWORD[((256-128))+rcx] - vpaddq ymm3,ymm3,ymm12 + vpaddq ymm3,ymm3,ymm12 ; correct %ymm3 vpaddq ymm8,ymm8,ymm0 mov rax,rbx @@ -918,9 +918,9 @@ vmovdqu ymm13,YMMWORD[((-8+64-128))+rsi] mov rax,r10 - vpblendd ymm9,ymm9,ymm14,0xfc + vpblendd ymm9,ymm9,ymm14,0xfc ; correct %ymm3 imul eax,r8d - vpaddq ymm4,ymm4,ymm9 + vpaddq ymm4,ymm4,ymm9 ; correct %ymm3 and eax,0x1fffffff imul rbx,QWORD[((16-128))+rsi] @@ -1108,7 +1108,7 @@ vpmuludq ymm13,ymm13,ymm10 vpbroadcastq ymm10,QWORD[32+r13] vpaddq ymm9,ymm9,ymm13 - add r13,32 + add r13,32 ; %r13++ vmovdqu ymm0,YMMWORD[((-24+32-128))+rcx] imul rax,QWORD[((-128))+rcx] @@ -1119,9 +1119,9 @@ vpmuludq ymm0,ymm0,ymm11 vmovq rbx,xmm10 vmovdqu ymm13,YMMWORD[((-24+96-128))+rcx] - vpaddq ymm0,ymm1,ymm0 + vpaddq ymm0,ymm1,ymm0 ; %ymm0==%ymm0 vpmuludq ymm12,ymm12,ymm11 - vmovdqu YMMWORD[rsp],ymm0 + vmovdqu YMMWORD[rsp],ymm0 ; transfer %r9-%r12 vpaddq ymm1,ymm2,ymm12 vmovdqu ymm0,YMMWORD[((-24+128-128))+rcx] vpmuludq ymm13,ymm13,ymm11 @@ -1289,7 +1289,7 @@ mov rbx,QWORD[((-8))+rax] - lea rsp,[rax] + lea rsp,[rax] ; restore %rsp $L$mul_1024_epilogue: mov rdi,QWORD[8+rsp] ;WIN64 epilogue @@ -1303,7 +1303,7 @@ rsaz_1024_red2norm_avx2: _CET_ENDBR - sub rdx,-128 + sub rdx,-128 ; size optimization xor rax,rax mov r8,QWORD[((-128))+rdx] mov r9,QWORD[((-120))+rdx] @@ -1316,7 +1316,7 @@ add rax,r8 add rax,r9 add rax,r10 - adc r11,0 + adc r11,0 ; consume eventual carry mov QWORD[rcx],rax mov rax,r11 mov r8,QWORD[((-104))+rdx] @@ -1327,7 +1327,7 @@ shr r10,12 add rax,r8 add rax,r9 - adc r10,0 + adc r10,0 ; consume eventual carry mov QWORD[8+rcx],rax mov rax,r10 mov r11,QWORD[((-88))+rdx] @@ -1338,7 +1338,7 @@ shr r9,18 add rax,r11 add rax,r8 - adc r9,0 + adc r9,0 ; consume eventual carry mov QWORD[16+rcx],rax mov rax,r9 mov r10,QWORD[((-72))+rdx] @@ -1349,7 +1349,7 @@ shr r8,24 add rax,r10 add rax,r11 - adc r8,0 + adc r8,0 ; consume eventual carry mov QWORD[24+rcx],rax mov rax,r8 mov r9,QWORD[((-56))+rdx] @@ -1363,7 +1363,7 @@ add rax,r9 add rax,r10 add rax,r11 - adc r8,0 + adc r8,0 ; consume eventual carry mov QWORD[32+rcx],rax mov rax,r8 mov r9,QWORD[((-32))+rdx] @@ -1374,7 +1374,7 @@ shr r11,7 add rax,r9 add rax,r10 - adc r11,0 + adc r11,0 ; consume eventual carry mov QWORD[40+rcx],rax mov rax,r11 mov r8,QWORD[((-16))+rdx] @@ -1385,7 +1385,7 @@ shr r10,13 add rax,r8 add rax,r9 - adc r10,0 + adc r10,0 ; consume eventual carry mov QWORD[48+rcx],rax mov rax,r10 mov r11,QWORD[rdx] @@ -1396,7 +1396,7 @@ shr r9,19 add rax,r11 add rax,r8 - adc r9,0 + adc r9,0 ; consume eventual carry mov QWORD[56+rcx],rax mov rax,r9 mov r10,QWORD[16+rdx] @@ -1407,7 +1407,7 @@ shr r8,25 add rax,r10 add rax,r11 - adc r8,0 + adc r8,0 ; consume eventual carry mov QWORD[64+rcx],rax mov rax,r8 mov r9,QWORD[32+rdx] @@ -1421,7 +1421,7 @@ add rax,r9 add rax,r10 add rax,r11 - adc r8,0 + adc r8,0 ; consume eventual carry mov QWORD[72+rcx],rax mov rax,r8 mov r9,QWORD[56+rdx] @@ -1432,7 +1432,7 @@ shr r11,8 add rax,r9 add rax,r10 - adc r11,0 + adc r11,0 ; consume eventual carry mov QWORD[80+rcx],rax mov rax,r11 mov r8,QWORD[72+rdx] @@ -1443,7 +1443,7 @@ shr r10,14 add rax,r8 add rax,r9 - adc r10,0 + adc r10,0 ; consume eventual carry mov QWORD[88+rcx],rax mov rax,r10 mov r11,QWORD[88+rdx] @@ -1454,7 +1454,7 @@ shr r9,20 add rax,r11 add rax,r8 - adc r9,0 + adc r9,0 ; consume eventual carry mov QWORD[96+rcx],rax mov rax,r9 mov r10,QWORD[104+rdx] @@ -1465,7 +1465,7 @@ shr r8,26 add rax,r10 add rax,r11 - adc r8,0 + adc r8,0 ; consume eventual carry mov QWORD[104+rcx],rax mov rax,r8 mov r9,QWORD[120+rdx] @@ -1479,7 +1479,7 @@ add rax,r9 add rax,r10 add rax,r11 - adc r8,0 + adc r8,0 ; consume eventual carry mov QWORD[112+rcx],rax mov rax,r8 mov r9,QWORD[144+rdx] @@ -1490,7 +1490,7 @@ shr r11,9 add rax,r9 add rax,r10 - adc r11,0 + adc r11,0 ; consume eventual carry mov QWORD[120+rcx],rax mov rax,r11 ret @@ -1503,17 +1503,17 @@ rsaz_1024_norm2red_avx2: _CET_ENDBR - sub rcx,-128 + sub rcx,-128 ; size optimization mov r8,QWORD[rdx] mov eax,0x1fffffff mov r9,QWORD[8+rdx] mov r11,r8 shr r11,0 - and r11,rax + and r11,rax ; &0x1fffffff mov QWORD[((-128))+rcx],r11 mov r10,r8 shr r10,29 - and r10,rax + and r10,rax ; &0x1fffffff mov QWORD[((-120))+rcx],r10 shrd r8,r9,58 and r8,rax @@ -1521,7 +1521,7 @@ mov r10,QWORD[16+rdx] mov r8,r9 shr r8,23 - and r8,rax + and r8,rax ; &0x1fffffff mov QWORD[((-104))+rcx],r8 shrd r9,r10,52 and r9,rax @@ -1529,7 +1529,7 @@ mov r11,QWORD[24+rdx] mov r9,r10 shr r9,17 - and r9,rax + and r9,rax ; &0x1fffffff mov QWORD[((-88))+rcx],r9 shrd r10,r11,46 and r10,rax @@ -1537,7 +1537,7 @@ mov r8,QWORD[32+rdx] mov r10,r11 shr r10,11 - and r10,rax + and r10,rax ; &0x1fffffff mov QWORD[((-72))+rcx],r10 shrd r11,r8,40 and r11,rax @@ -1545,11 +1545,11 @@ mov r9,QWORD[40+rdx] mov r11,r8 shr r11,5 - and r11,rax + and r11,rax ; &0x1fffffff mov QWORD[((-56))+rcx],r11 mov r10,r8 shr r10,34 - and r10,rax + and r10,rax ; &0x1fffffff mov QWORD[((-48))+rcx],r10 shrd r8,r9,63 and r8,rax @@ -1557,7 +1557,7 @@ mov r10,QWORD[48+rdx] mov r8,r9 shr r8,28 - and r8,rax + and r8,rax ; &0x1fffffff mov QWORD[((-32))+rcx],r8 shrd r9,r10,57 and r9,rax @@ -1565,7 +1565,7 @@ mov r11,QWORD[56+rdx] mov r9,r10 shr r9,22 - and r9,rax + and r9,rax ; &0x1fffffff mov QWORD[((-16))+rcx],r9 shrd r10,r11,51 and r10,rax @@ -1573,7 +1573,7 @@ mov r8,QWORD[64+rdx] mov r10,r11 shr r10,16 - and r10,rax + and r10,rax ; &0x1fffffff mov QWORD[rcx],r10 shrd r11,r8,45 and r11,rax @@ -1581,7 +1581,7 @@ mov r9,QWORD[72+rdx] mov r11,r8 shr r11,10 - and r11,rax + and r11,rax ; &0x1fffffff mov QWORD[16+rcx],r11 shrd r8,r9,39 and r8,rax @@ -1589,11 +1589,11 @@ mov r10,QWORD[80+rdx] mov r8,r9 shr r8,4 - and r8,rax + and r8,rax ; &0x1fffffff mov QWORD[32+rcx],r8 mov r11,r9 shr r11,33 - and r11,rax + and r11,rax ; &0x1fffffff mov QWORD[40+rcx],r11 shrd r9,r10,62 and r9,rax @@ -1601,7 +1601,7 @@ mov r11,QWORD[88+rdx] mov r9,r10 shr r9,27 - and r9,rax + and r9,rax ; &0x1fffffff mov QWORD[56+rcx],r9 shrd r10,r11,56 and r10,rax @@ -1609,7 +1609,7 @@ mov r8,QWORD[96+rdx] mov r10,r11 shr r10,21 - and r10,rax + and r10,rax ; &0x1fffffff mov QWORD[72+rcx],r10 shrd r11,r8,50 and r11,rax @@ -1617,7 +1617,7 @@ mov r9,QWORD[104+rdx] mov r11,r8 shr r11,15 - and r11,rax + and r11,rax ; &0x1fffffff mov QWORD[88+rcx],r11 shrd r8,r9,44 and r8,rax @@ -1625,7 +1625,7 @@ mov r10,QWORD[112+rdx] mov r8,r9 shr r8,9 - and r8,rax + and r8,rax ; &0x1fffffff mov QWORD[104+rcx],r8 shrd r9,r10,38 and r9,rax @@ -1633,11 +1633,11 @@ mov r11,QWORD[120+rdx] mov r9,r10 shr r9,3 - and r9,rax + and r9,rax ; &0x1fffffff mov QWORD[120+rcx],r9 mov r8,r10 shr r8,32 - and r8,rax + and r8,rax ; &0x1fffffff mov QWORD[128+rcx],r8 shrd r10,r11,61 and r10,rax @@ -1645,12 +1645,12 @@ xor r8,r8 mov r10,r11 shr r10,26 - and r10,rax + and r10,rax ; &0x1fffffff mov QWORD[144+rcx],r10 shrd r11,r8,55 and r11,rax mov QWORD[152+rcx],r11 - mov QWORD[160+rcx],r8 + mov QWORD[160+rcx],r8 ; zero mov QWORD[168+rcx],r8 mov QWORD[176+rcx],r8 mov QWORD[184+rcx],r8 @@ -1696,22 +1696,22 @@ lea rax,[((-136))+rsp] $L$SEH_begin_rsaz_1024_gather5: - - DB 0x48,0x8d,0x60,0xe0 - DB 0xc5,0xf8,0x29,0x70,0xe0 - DB 0xc5,0xf8,0x29,0x78,0xf0 - DB 0xc5,0x78,0x29,0x40,0x00 - DB 0xc5,0x78,0x29,0x48,0x10 - DB 0xc5,0x78,0x29,0x50,0x20 - DB 0xc5,0x78,0x29,0x58,0x30 - DB 0xc5,0x78,0x29,0x60,0x40 - DB 0xc5,0x78,0x29,0x68,0x50 - DB 0xc5,0x78,0x29,0x70,0x60 - DB 0xc5,0x78,0x29,0x78,0x70 +; I can't trust assembler to use specific encoding:-( + DB 0x48,0x8d,0x60,0xe0 ; lea -0x20(%rax),%rsp + DB 0xc5,0xf8,0x29,0x70,0xe0 ; vmovaps %xmm6,-0x20(%rax) + DB 0xc5,0xf8,0x29,0x78,0xf0 ; vmovaps %xmm7,-0x10(%rax) + DB 0xc5,0x78,0x29,0x40,0x00 ; vmovaps %xmm8,0(%rax) + DB 0xc5,0x78,0x29,0x48,0x10 ; vmovaps %xmm9,0x10(%rax) + DB 0xc5,0x78,0x29,0x50,0x20 ; vmovaps %xmm10,0x20(%rax) + DB 0xc5,0x78,0x29,0x58,0x30 ; vmovaps %xmm11,0x30(%rax) + DB 0xc5,0x78,0x29,0x60,0x40 ; vmovaps %xmm12,0x40(%rax) + DB 0xc5,0x78,0x29,0x68,0x50 ; vmovaps %xmm13,0x50(%rax) + DB 0xc5,0x78,0x29,0x70,0x60 ; vmovaps %xmm14,0x60(%rax) + DB 0xc5,0x78,0x29,0x78,0x70 ; vmovaps %xmm15,0x70(%rax) lea rsp,[((-256))+rsp] and rsp,-32 lea r10,[$L$inc] - lea rax,[((-128))+rsp] + lea rax,[((-128))+rsp] ; control u-op density vmovd xmm4,r8d vmovdqa ymm0,YMMWORD[r10] @@ -1758,7 +1758,7 @@ vpcmpeqd ymm14,ymm14,ymm4 vpcmpeqd ymm15,ymm15,ymm4 - vmovdqa ymm7,YMMWORD[((-32))+r10] + vmovdqa ymm7,YMMWORD[((-32))+r10] ; .Lgather_permd lea rdx,[128+rdx] mov r8d,9 @@ -1804,7 +1804,7 @@ vpor ymm5,ymm5,ymm3 vpor ymm4,ymm4,ymm5 - vextracti128 xmm5,ymm4,1 + vextracti128 xmm5,ymm4,1 ; upper half is cleared vpor xmm5,xmm5,xmm4 vpermd ymm5,ymm7,ymm5 vmovdqu YMMWORD[rcx],ymm5 @@ -1863,27 +1863,27 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$common_seh_tail - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail - mov rbp,QWORD[160+r8] + mov rbp,QWORD[160+r8] ; pull context->Rbp - mov r10d,DWORD[8+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[8+r11] ; HandlerData[2] + lea r10,[r10*1+rsi] ; "in tail" label + cmp rbx,r10 ; context->Rip>="in tail" label cmovc rax,rbp mov r15,QWORD[((-48))+rax] @@ -1899,38 +1899,38 @@ mov QWORD[160+r8],rbp mov QWORD[144+r8],rbx - lea rsi,[((-216))+rax] - lea rdi,[512+r8] - mov ecx,20 - DD 0xa548f3fc + lea rsi,[((-216))+rax] ; %xmm save area + lea rdi,[512+r8] ; & context.Xmm6 + mov ecx,20 ; 10*sizeof(%xmm0)/sizeof(%rax) + DD 0xa548f3fc ; cld; rep movsq $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -1971,18 +1971,18 @@ DD 0 $L$SEH_info_rsaz_1024_gather5: DB 0x01,0x36,0x17,0x0b - DB 0x36,0xf8,0x09,0x00 - DB 0x31,0xe8,0x08,0x00 - DB 0x2c,0xd8,0x07,0x00 - DB 0x27,0xc8,0x06,0x00 - DB 0x22,0xb8,0x05,0x00 - DB 0x1d,0xa8,0x04,0x00 - DB 0x18,0x98,0x03,0x00 - DB 0x13,0x88,0x02,0x00 - DB 0x0e,0x78,0x01,0x00 - DB 0x09,0x68,0x00,0x00 - DB 0x04,0x01,0x15,0x00 - DB 0x00,0xb3,0x00,0x00 + DB 0x36,0xf8,0x09,0x00 ; vmovaps 0x90(rsp),xmm15 + DB 0x31,0xe8,0x08,0x00 ; vmovaps 0x80(rsp),xmm14 + DB 0x2c,0xd8,0x07,0x00 ; vmovaps 0x70(rsp),xmm13 + DB 0x27,0xc8,0x06,0x00 ; vmovaps 0x60(rsp),xmm12 + DB 0x22,0xb8,0x05,0x00 ; vmovaps 0x50(rsp),xmm11 + DB 0x1d,0xa8,0x04,0x00 ; vmovaps 0x40(rsp),xmm10 + DB 0x18,0x98,0x03,0x00 ; vmovaps 0x30(rsp),xmm9 + DB 0x13,0x88,0x02,0x00 ; vmovaps 0x20(rsp),xmm8 + DB 0x0e,0x78,0x01,0x00 ; vmovaps 0x10(rsp),xmm7 + DB 0x09,0x68,0x00,0x00 ; vmovaps 0x00(rsp),xmm6 + DB 0x04,0x01,0x15,0x00 ; sub rsp,0xa8 + DB 0x00,0xb3,0x00,0x00 ; set_frame r11 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/bcm/sha1-x86_64-apple.S b/gen/bcm/sha1-x86_64-apple.S index 41e95c9..64ab491 100644 --- a/gen/bcm/sha1-x86_64-apple.S +++ b/gen/bcm/sha1-x86_64-apple.S
@@ -25,11 +25,11 @@ pushq %r14 - movq %rdi,%r8 + movq %rdi,%r8 // reassigned argument subq $72,%rsp - movq %rsi,%r9 + movq %rsi,%r9 // reassigned argument andq $-64,%rsp - movq %rdx,%r10 + movq %rdx,%r10 // reassigned argument movq %rax,64(%rsp) L$prologue: @@ -1253,32 +1253,32 @@ _CET_ENDBR movdqu (%rdi),%xmm0 movd 16(%rdi),%xmm1 - movdqa K_XX_XX+160(%rip),%xmm3 + movdqa K_XX_XX+160(%rip),%xmm3 // byte-n-word swap movdqu (%rsi),%xmm4 - pshufd $27,%xmm0,%xmm0 + pshufd $27,%xmm0,%xmm0 // flip word order movdqu 16(%rsi),%xmm5 - pshufd $27,%xmm1,%xmm1 + pshufd $27,%xmm1,%xmm1 // flip word order movdqu 32(%rsi),%xmm6 pshufb %xmm3,%xmm4 movdqu 48(%rsi),%xmm7 pshufb %xmm3,%xmm5 pshufb %xmm3,%xmm6 - movdqa %xmm1,%xmm9 + movdqa %xmm1,%xmm9 // offload %xmm1 pshufb %xmm3,%xmm7 jmp L$oop_shaext .p2align 4 L$oop_shaext: decq %rdx - leaq 64(%rsi),%r8 + leaq 64(%rsi),%r8 // next input block paddd %xmm4,%xmm1 cmovneq %r8,%rsi prefetcht0 512(%rsi) - movdqa %xmm0,%xmm8 + movdqa %xmm0,%xmm8 // offload %xmm0 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $0,%xmm1,%xmm0 + sha1rnds4 $0,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1291,7 +1291,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $0,%xmm1,%xmm0 + sha1rnds4 $0,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1304,7 +1304,7 @@ sha1msg2 %xmm6,%xmm7 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $0,%xmm1,%xmm0 + sha1rnds4 $0,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1317,7 +1317,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $1,%xmm1,%xmm0 + sha1rnds4 $1,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1330,7 +1330,7 @@ sha1msg2 %xmm6,%xmm7 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $1,%xmm1,%xmm0 + sha1rnds4 $1,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1343,7 +1343,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $2,%xmm1,%xmm0 + sha1rnds4 $2,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1356,7 +1356,7 @@ sha1msg2 %xmm6,%xmm7 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $2,%xmm1,%xmm0 + sha1rnds4 $2,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1369,7 +1369,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $2,%xmm1,%xmm0 + sha1rnds4 $2,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1382,30 +1382,30 @@ sha1msg2 %xmm6,%xmm7 movdqu (%rsi),%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $3,%xmm1,%xmm0 + sha1rnds4 $3,%xmm1,%xmm0 // 64-67 sha1nexte %xmm5,%xmm2 movdqu 16(%rsi),%xmm5 pshufb %xmm3,%xmm4 movdqa %xmm0,%xmm1 - sha1rnds4 $3,%xmm2,%xmm0 + sha1rnds4 $3,%xmm2,%xmm0 // 68-71 sha1nexte %xmm6,%xmm1 movdqu 32(%rsi),%xmm6 pshufb %xmm3,%xmm5 movdqa %xmm0,%xmm2 - sha1rnds4 $3,%xmm1,%xmm0 + sha1rnds4 $3,%xmm1,%xmm0 // 72-75 sha1nexte %xmm7,%xmm2 movdqu 48(%rsi),%xmm7 pshufb %xmm3,%xmm6 movdqa %xmm0,%xmm1 - sha1rnds4 $3,%xmm2,%xmm0 + sha1rnds4 $3,%xmm2,%xmm0 // 76-79 sha1nexte %xmm9,%xmm1 pshufb %xmm3,%xmm7 paddd %xmm8,%xmm0 - movdqa %xmm1,%xmm9 + movdqa %xmm1,%xmm9 // offload %xmm1 jnz L$oop_shaext @@ -1424,7 +1424,7 @@ _sha1_block_data_order_ssse3: _CET_ENDBR - movq %rsp,%r11 + movq %rsp,%r11 // frame pointer pushq %rbx @@ -1432,46 +1432,46 @@ pushq %r12 - pushq %r13 + pushq %r13 // redundant, done to share Win64 SE handler pushq %r14 leaq -64(%rsp),%rsp andq $-64,%rsp - movq %rdi,%r8 - movq %rsi,%r9 - movq %rdx,%r10 + movq %rdi,%r8 // reassigned argument + movq %rsi,%r9 // reassigned argument + movq %rdx,%r10 // reassigned argument shlq $6,%r10 addq %r9,%r10 leaq K_XX_XX+64(%rip),%r14 - movl 0(%r8),%eax + movl 0(%r8),%eax // load context movl 4(%r8),%ebx movl 8(%r8),%ecx movl 12(%r8),%edx - movl %ebx,%esi + movl %ebx,%esi // magic seed movl 16(%r8),%ebp movl %ecx,%edi xorl %edx,%edi andl %edi,%esi - movdqa 64(%r14),%xmm6 - movdqa -64(%r14),%xmm9 - movdqu 0(%r9),%xmm0 + movdqa 64(%r14),%xmm6 // pbswap mask + movdqa -64(%r14),%xmm9 // K_00_19 + movdqu 0(%r9),%xmm0 // load input to %xmm[0-3] movdqu 16(%r9),%xmm1 movdqu 32(%r9),%xmm2 movdqu 48(%r9),%xmm3 - pshufb %xmm6,%xmm0 + pshufb %xmm6,%xmm0 // byte swap pshufb %xmm6,%xmm1 pshufb %xmm6,%xmm2 addq $64,%r9 - paddd %xmm9,%xmm0 + paddd %xmm9,%xmm0 // add K_00_19 pshufb %xmm6,%xmm3 paddd %xmm9,%xmm1 paddd %xmm9,%xmm2 - movdqa %xmm0,0(%rsp) - psubd %xmm9,%xmm0 + movdqa %xmm0,0(%rsp) // X[]+K xfer to IALU + psubd %xmm9,%xmm0 // restore X[] movdqa %xmm1,16(%rsp) psubd %xmm9,%xmm1 movdqa %xmm2,32(%rsp) @@ -2468,14 +2468,14 @@ addl %edi,%eax rorl $7,%ecx addl %ebx,%eax - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx addl 12(%r8),%edx movl %eax,0(%r8) addl 16(%r8),%ebp movl %esi,4(%r8) - movl %esi,%ebx + movl %esi,%ebx // magic seed movl %ecx,8(%r8) movl %ecx,%edi movl %edx,12(%r8) @@ -2581,7 +2581,7 @@ addl %edi,%eax rorl $7,%ecx addl %ebx,%eax - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx movl %eax,0(%r8) @@ -2623,46 +2623,46 @@ pushq %r12 - pushq %r13 + pushq %r13 // redundant, done to share Win64 SE handler pushq %r14 leaq -64(%rsp),%rsp vzeroupper andq $-64,%rsp - movq %rdi,%r8 - movq %rsi,%r9 - movq %rdx,%r10 + movq %rdi,%r8 // reassigned argument + movq %rsi,%r9 // reassigned argument + movq %rdx,%r10 // reassigned argument shlq $6,%r10 addq %r9,%r10 leaq K_XX_XX+64(%rip),%r14 - movl 0(%r8),%eax + movl 0(%r8),%eax // load context movl 4(%r8),%ebx movl 8(%r8),%ecx movl 12(%r8),%edx - movl %ebx,%esi + movl %ebx,%esi // magic seed movl 16(%r8),%ebp movl %ecx,%edi xorl %edx,%edi andl %edi,%esi - vmovdqa 64(%r14),%xmm6 - vmovdqa -64(%r14),%xmm11 - vmovdqu 0(%r9),%xmm0 + vmovdqa 64(%r14),%xmm6 // pbswap mask + vmovdqa -64(%r14),%xmm11 // K_00_19 + vmovdqu 0(%r9),%xmm0 // load input to %xmm[0-3] vmovdqu 16(%r9),%xmm1 vmovdqu 32(%r9),%xmm2 vmovdqu 48(%r9),%xmm3 - vpshufb %xmm6,%xmm0,%xmm0 + vpshufb %xmm6,%xmm0,%xmm0 // byte swap addq $64,%r9 vpshufb %xmm6,%xmm1,%xmm1 vpshufb %xmm6,%xmm2,%xmm2 vpshufb %xmm6,%xmm3,%xmm3 - vpaddd %xmm11,%xmm0,%xmm4 + vpaddd %xmm11,%xmm0,%xmm4 // add K_00_19 vpaddd %xmm11,%xmm1,%xmm5 vpaddd %xmm11,%xmm2,%xmm6 - vmovdqa %xmm4,0(%rsp) + vmovdqa %xmm4,0(%rsp) // X[]+K xfer to IALU vmovdqa %xmm5,16(%rsp) vmovdqa %xmm6,32(%rsp) jmp L$oop_avx @@ -3597,14 +3597,14 @@ addl %edi,%eax shrdl $7,%ecx,%ecx addl %ebx,%eax - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx addl 12(%r8),%edx movl %eax,0(%r8) addl 16(%r8),%ebp movl %esi,4(%r8) - movl %esi,%ebx + movl %esi,%ebx // magic seed movl %ecx,8(%r8) movl %ecx,%edi movl %edx,12(%r8) @@ -3712,7 +3712,7 @@ addl %ebx,%eax vzeroupper - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx movl %eax,0(%r8) @@ -3759,9 +3759,9 @@ pushq %r14 vzeroupper - movq %rdi,%r8 - movq %rsi,%r9 - movq %rdx,%r10 + movq %rdi,%r8 // reassigned argument + movq %rsi,%r9 // reassigned argument + movq %rdx,%r10 // reassigned argument leaq -640(%rsp),%rsp shlq $6,%r10 @@ -3770,14 +3770,14 @@ addq %r9,%r10 leaq K_XX_XX+64(%rip),%r14 - movl 0(%r8),%eax + movl 0(%r8),%eax // load context cmpq %r10,%r13 - cmovaeq %r9,%r13 + cmovaeq %r9,%r13 // next or same block movl 4(%r8),%ebp movl 8(%r8),%ecx movl 12(%r8),%edx movl 16(%r8),%esi - vmovdqu 64(%r14),%ymm6 + vmovdqu 64(%r14),%ymm6 // pbswap mask vmovdqu (%r9),%xmm0 vmovdqu 16(%r9),%xmm1 @@ -3791,12 +3791,12 @@ vpshufb %ymm6,%ymm1,%ymm1 vinserti128 $1,48(%r13),%ymm3,%ymm3 vpshufb %ymm6,%ymm2,%ymm2 - vmovdqu -64(%r14),%ymm11 + vmovdqu -64(%r14),%ymm11 // K_00_19 vpshufb %ymm6,%ymm3,%ymm3 - vpaddd %ymm11,%ymm0,%ymm4 + vpaddd %ymm11,%ymm0,%ymm4 // add K_00_19 vpaddd %ymm11,%ymm1,%ymm5 - vmovdqu %ymm4,0(%rsp) + vmovdqu %ymm4,0(%rsp) // X[]+K xfer to IALU vpaddd %ymm11,%ymm2,%ymm6 vmovdqu %ymm5,32(%rsp) vpaddd %ymm11,%ymm3,%ymm7 @@ -3871,7 +3871,7 @@ andnl %edx,%ebp,%edi andl %ecx,%ebp xorl %edi,%ebp - jmp L$align32_1 + jmp L$align32_1 // see "Decoded ICache" in manual .p2align 5 L$align32_1: vpalignr $8,%ymm6,%ymm7,%ymm8 @@ -4250,7 +4250,7 @@ xorl %esi,%ecx addl %r12d,%ebx andl %edi,%ecx - jmp L$align32_2 + jmp L$align32_2 // see "Decoded ICache" in manual .p2align 5 L$align32_2: vpalignr $8,%ymm6,%ymm7,%ymm8 @@ -4626,38 +4626,38 @@ rorxl $27,%esi,%r12d addl %r12d,%edx leaq 128(%r9),%r13 - leaq 128(%r9),%rdi + leaq 128(%r9),%rdi // borrow %edi cmpq %r10,%r13 - cmovaeq %r9,%r13 + cmovaeq %r9,%r13 // next or previous block - - addl 0(%r8),%edx +// output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + addl 0(%r8),%edx // update context addl 4(%r8),%esi addl 8(%r8),%ebp movl %edx,0(%r8) addl 12(%r8),%ebx movl %esi,4(%r8) - movl %edx,%eax + movl %edx,%eax // A=d addl 16(%r8),%ecx movl %ebp,%r12d movl %ebp,8(%r8) - movl %ebx,%edx - + movl %ebx,%edx // D=b +// xchg %ecx,%ebp # F=c, C=f movl %ebx,12(%r8) - movl %esi,%ebp + movl %esi,%ebp // F=e movl %ecx,16(%r8) - - movl %ecx,%esi - movl %r12d,%ecx - +// mov %ebp,16(%r8) + movl %ecx,%esi // E=c + movl %r12d,%ecx // C=f +// xchg %ebp,%esi # E=c, F=e cmpq %r10,%r9 je L$done_avx2 - vmovdqu 64(%r14),%ymm6 - cmpq %r10,%rdi + vmovdqu 64(%r14),%ymm6 // pbswap mask + cmpq %r10,%rdi // borrowed %edi ja L$ast_avx2 - vmovdqu -64(%rdi),%xmm0 + vmovdqu -64(%rdi),%xmm0 // low part of %ymm0 vmovdqu -48(%rdi),%xmm1 vmovdqu -32(%rdi),%xmm2 vmovdqu -16(%rdi),%xmm3 @@ -5037,7 +5037,7 @@ xorl %eax,%edx addl %r12d,%ecx andl %edi,%edx - jmp L$align32_3 + jmp L$align32_3 // see "Decoded ICache" in manual .p2align 5 L$align32_3: vmovdqu %ymm6,64(%rsp) @@ -5392,26 +5392,26 @@ addl %r12d,%edx leaq 128(%rsp),%r13 - - addl 0(%r8),%edx +// output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + addl 0(%r8),%edx // update context addl 4(%r8),%esi addl 8(%r8),%ebp movl %edx,0(%r8) addl 12(%r8),%ebx movl %esi,4(%r8) - movl %edx,%eax + movl %edx,%eax // A=d addl 16(%r8),%ecx movl %ebp,%r12d movl %ebp,8(%r8) - movl %ebx,%edx - + movl %ebx,%edx // D=b +// xchg %ecx,%ebp # F=c, C=f movl %ebx,12(%r8) - movl %esi,%ebp + movl %esi,%ebp // F=e movl %ecx,16(%r8) - - movl %ecx,%esi - movl %r12d,%ecx - +// mov %ebp,16(%r8) + movl %ecx,%esi // E=c + movl %r12d,%ecx // C=f +// xchg %ebp,%esi # E=c, F=e cmpq %r10,%r9 jbe L$oop_avx2 @@ -5437,16 +5437,16 @@ .section __DATA,__const .p2align 6 K_XX_XX: -.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 -.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 -.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 -.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 -.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc -.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc -.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 -.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 -.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f -.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 // K_00_19 +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 // K_00_19 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 // K_20_39 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 // K_20_39 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc // K_40_59 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc // K_40_59 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 // K_60_79 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 // K_60_79 +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f // pbswap mask +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f // pbswap mask .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 .byte 83,72,65,49,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 .p2align 6
diff --git a/gen/bcm/sha1-x86_64-linux.S b/gen/bcm/sha1-x86_64-linux.S index 1f4807a..005d4c3 100644 --- a/gen/bcm/sha1-x86_64-linux.S +++ b/gen/bcm/sha1-x86_64-linux.S
@@ -25,11 +25,11 @@ .cfi_offset %r13,-40 pushq %r14 .cfi_offset %r14,-48 - movq %rdi,%r8 + movq %rdi,%r8 // reassigned argument subq $72,%rsp - movq %rsi,%r9 + movq %rsi,%r9 // reassigned argument andq $-64,%rsp - movq %rdx,%r10 + movq %rdx,%r10 // reassigned argument movq %rax,64(%rsp) .cfi_escape 0x0f,0x06,0x77,0xc0,0x00,0x06,0x23,0x08 .Lprologue: @@ -1252,32 +1252,32 @@ _CET_ENDBR movdqu (%rdi),%xmm0 movd 16(%rdi),%xmm1 - movdqa K_XX_XX+160(%rip),%xmm3 + movdqa K_XX_XX+160(%rip),%xmm3 // byte-n-word swap movdqu (%rsi),%xmm4 - pshufd $27,%xmm0,%xmm0 + pshufd $27,%xmm0,%xmm0 // flip word order movdqu 16(%rsi),%xmm5 - pshufd $27,%xmm1,%xmm1 + pshufd $27,%xmm1,%xmm1 // flip word order movdqu 32(%rsi),%xmm6 pshufb %xmm3,%xmm4 movdqu 48(%rsi),%xmm7 pshufb %xmm3,%xmm5 pshufb %xmm3,%xmm6 - movdqa %xmm1,%xmm9 + movdqa %xmm1,%xmm9 // offload %xmm1 pshufb %xmm3,%xmm7 jmp .Loop_shaext .align 16 .Loop_shaext: decq %rdx - leaq 64(%rsi),%r8 + leaq 64(%rsi),%r8 // next input block paddd %xmm4,%xmm1 cmovneq %r8,%rsi prefetcht0 512(%rsi) - movdqa %xmm0,%xmm8 + movdqa %xmm0,%xmm8 // offload %xmm0 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $0,%xmm1,%xmm0 + sha1rnds4 $0,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1290,7 +1290,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $0,%xmm1,%xmm0 + sha1rnds4 $0,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1303,7 +1303,7 @@ sha1msg2 %xmm6,%xmm7 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $0,%xmm1,%xmm0 + sha1rnds4 $0,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1316,7 +1316,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $1,%xmm1,%xmm0 + sha1rnds4 $1,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1329,7 +1329,7 @@ sha1msg2 %xmm6,%xmm7 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $1,%xmm1,%xmm0 + sha1rnds4 $1,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1342,7 +1342,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $2,%xmm1,%xmm0 + sha1rnds4 $2,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1355,7 +1355,7 @@ sha1msg2 %xmm6,%xmm7 sha1msg1 %xmm5,%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $2,%xmm1,%xmm0 + sha1rnds4 $2,%xmm1,%xmm0 // 0-3... sha1nexte %xmm5,%xmm2 pxor %xmm6,%xmm4 sha1msg1 %xmm6,%xmm5 @@ -1368,7 +1368,7 @@ sha1msg2 %xmm4,%xmm5 sha1msg1 %xmm7,%xmm6 movdqa %xmm0,%xmm2 - sha1rnds4 $2,%xmm1,%xmm0 + sha1rnds4 $2,%xmm1,%xmm0 // 0-3... sha1nexte %xmm7,%xmm2 pxor %xmm4,%xmm6 sha1msg1 %xmm4,%xmm7 @@ -1381,30 +1381,30 @@ sha1msg2 %xmm6,%xmm7 movdqu (%rsi),%xmm4 movdqa %xmm0,%xmm2 - sha1rnds4 $3,%xmm1,%xmm0 + sha1rnds4 $3,%xmm1,%xmm0 // 64-67 sha1nexte %xmm5,%xmm2 movdqu 16(%rsi),%xmm5 pshufb %xmm3,%xmm4 movdqa %xmm0,%xmm1 - sha1rnds4 $3,%xmm2,%xmm0 + sha1rnds4 $3,%xmm2,%xmm0 // 68-71 sha1nexte %xmm6,%xmm1 movdqu 32(%rsi),%xmm6 pshufb %xmm3,%xmm5 movdqa %xmm0,%xmm2 - sha1rnds4 $3,%xmm1,%xmm0 + sha1rnds4 $3,%xmm1,%xmm0 // 72-75 sha1nexte %xmm7,%xmm2 movdqu 48(%rsi),%xmm7 pshufb %xmm3,%xmm6 movdqa %xmm0,%xmm1 - sha1rnds4 $3,%xmm2,%xmm0 + sha1rnds4 $3,%xmm2,%xmm0 // 76-79 sha1nexte %xmm9,%xmm1 pshufb %xmm3,%xmm7 paddd %xmm8,%xmm0 - movdqa %xmm1,%xmm9 + movdqa %xmm1,%xmm9 // offload %xmm1 jnz .Loop_shaext @@ -1422,7 +1422,7 @@ sha1_block_data_order_ssse3: .cfi_startproc _CET_ENDBR - movq %rsp,%r11 + movq %rsp,%r11 // frame pointer .cfi_def_cfa_register %r11 pushq %rbx .cfi_offset %rbx,-16 @@ -1430,46 +1430,46 @@ .cfi_offset %rbp,-24 pushq %r12 .cfi_offset %r12,-32 - pushq %r13 + pushq %r13 // redundant, done to share Win64 SE handler .cfi_offset %r13,-40 pushq %r14 .cfi_offset %r14,-48 leaq -64(%rsp),%rsp andq $-64,%rsp - movq %rdi,%r8 - movq %rsi,%r9 - movq %rdx,%r10 + movq %rdi,%r8 // reassigned argument + movq %rsi,%r9 // reassigned argument + movq %rdx,%r10 // reassigned argument shlq $6,%r10 addq %r9,%r10 leaq K_XX_XX+64(%rip),%r14 - movl 0(%r8),%eax + movl 0(%r8),%eax // load context movl 4(%r8),%ebx movl 8(%r8),%ecx movl 12(%r8),%edx - movl %ebx,%esi + movl %ebx,%esi // magic seed movl 16(%r8),%ebp movl %ecx,%edi xorl %edx,%edi andl %edi,%esi - movdqa 64(%r14),%xmm6 - movdqa -64(%r14),%xmm9 - movdqu 0(%r9),%xmm0 + movdqa 64(%r14),%xmm6 // pbswap mask + movdqa -64(%r14),%xmm9 // K_00_19 + movdqu 0(%r9),%xmm0 // load input to %xmm[0-3] movdqu 16(%r9),%xmm1 movdqu 32(%r9),%xmm2 movdqu 48(%r9),%xmm3 - pshufb %xmm6,%xmm0 + pshufb %xmm6,%xmm0 // byte swap pshufb %xmm6,%xmm1 pshufb %xmm6,%xmm2 addq $64,%r9 - paddd %xmm9,%xmm0 + paddd %xmm9,%xmm0 // add K_00_19 pshufb %xmm6,%xmm3 paddd %xmm9,%xmm1 paddd %xmm9,%xmm2 - movdqa %xmm0,0(%rsp) - psubd %xmm9,%xmm0 + movdqa %xmm0,0(%rsp) // X[]+K xfer to IALU + psubd %xmm9,%xmm0 // restore X[] movdqa %xmm1,16(%rsp) psubd %xmm9,%xmm1 movdqa %xmm2,32(%rsp) @@ -2466,14 +2466,14 @@ addl %edi,%eax rorl $7,%ecx addl %ebx,%eax - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx addl 12(%r8),%edx movl %eax,0(%r8) addl 16(%r8),%ebp movl %esi,4(%r8) - movl %esi,%ebx + movl %esi,%ebx // magic seed movl %ecx,8(%r8) movl %ecx,%edi movl %edx,12(%r8) @@ -2579,7 +2579,7 @@ addl %edi,%eax rorl $7,%ecx addl %ebx,%eax - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx movl %eax,0(%r8) @@ -2620,46 +2620,46 @@ .cfi_offset %rbp,-24 pushq %r12 .cfi_offset %r12,-32 - pushq %r13 + pushq %r13 // redundant, done to share Win64 SE handler .cfi_offset %r13,-40 pushq %r14 .cfi_offset %r14,-48 leaq -64(%rsp),%rsp vzeroupper andq $-64,%rsp - movq %rdi,%r8 - movq %rsi,%r9 - movq %rdx,%r10 + movq %rdi,%r8 // reassigned argument + movq %rsi,%r9 // reassigned argument + movq %rdx,%r10 // reassigned argument shlq $6,%r10 addq %r9,%r10 leaq K_XX_XX+64(%rip),%r14 - movl 0(%r8),%eax + movl 0(%r8),%eax // load context movl 4(%r8),%ebx movl 8(%r8),%ecx movl 12(%r8),%edx - movl %ebx,%esi + movl %ebx,%esi // magic seed movl 16(%r8),%ebp movl %ecx,%edi xorl %edx,%edi andl %edi,%esi - vmovdqa 64(%r14),%xmm6 - vmovdqa -64(%r14),%xmm11 - vmovdqu 0(%r9),%xmm0 + vmovdqa 64(%r14),%xmm6 // pbswap mask + vmovdqa -64(%r14),%xmm11 // K_00_19 + vmovdqu 0(%r9),%xmm0 // load input to %xmm[0-3] vmovdqu 16(%r9),%xmm1 vmovdqu 32(%r9),%xmm2 vmovdqu 48(%r9),%xmm3 - vpshufb %xmm6,%xmm0,%xmm0 + vpshufb %xmm6,%xmm0,%xmm0 // byte swap addq $64,%r9 vpshufb %xmm6,%xmm1,%xmm1 vpshufb %xmm6,%xmm2,%xmm2 vpshufb %xmm6,%xmm3,%xmm3 - vpaddd %xmm11,%xmm0,%xmm4 + vpaddd %xmm11,%xmm0,%xmm4 // add K_00_19 vpaddd %xmm11,%xmm1,%xmm5 vpaddd %xmm11,%xmm2,%xmm6 - vmovdqa %xmm4,0(%rsp) + vmovdqa %xmm4,0(%rsp) // X[]+K xfer to IALU vmovdqa %xmm5,16(%rsp) vmovdqa %xmm6,32(%rsp) jmp .Loop_avx @@ -3594,14 +3594,14 @@ addl %edi,%eax shrdl $7,%ecx,%ecx addl %ebx,%eax - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx addl 12(%r8),%edx movl %eax,0(%r8) addl 16(%r8),%ebp movl %esi,4(%r8) - movl %esi,%ebx + movl %esi,%ebx // magic seed movl %ecx,8(%r8) movl %ecx,%edi movl %edx,12(%r8) @@ -3709,7 +3709,7 @@ addl %ebx,%eax vzeroupper - addl 0(%r8),%eax + addl 0(%r8),%eax // update context addl 4(%r8),%esi addl 8(%r8),%ecx movl %eax,0(%r8) @@ -3755,9 +3755,9 @@ pushq %r14 .cfi_offset %r14,-48 vzeroupper - movq %rdi,%r8 - movq %rsi,%r9 - movq %rdx,%r10 + movq %rdi,%r8 // reassigned argument + movq %rsi,%r9 // reassigned argument + movq %rdx,%r10 // reassigned argument leaq -640(%rsp),%rsp shlq $6,%r10 @@ -3766,14 +3766,14 @@ addq %r9,%r10 leaq K_XX_XX+64(%rip),%r14 - movl 0(%r8),%eax + movl 0(%r8),%eax // load context cmpq %r10,%r13 - cmovaeq %r9,%r13 + cmovaeq %r9,%r13 // next or same block movl 4(%r8),%ebp movl 8(%r8),%ecx movl 12(%r8),%edx movl 16(%r8),%esi - vmovdqu 64(%r14),%ymm6 + vmovdqu 64(%r14),%ymm6 // pbswap mask vmovdqu (%r9),%xmm0 vmovdqu 16(%r9),%xmm1 @@ -3787,12 +3787,12 @@ vpshufb %ymm6,%ymm1,%ymm1 vinserti128 $1,48(%r13),%ymm3,%ymm3 vpshufb %ymm6,%ymm2,%ymm2 - vmovdqu -64(%r14),%ymm11 + vmovdqu -64(%r14),%ymm11 // K_00_19 vpshufb %ymm6,%ymm3,%ymm3 - vpaddd %ymm11,%ymm0,%ymm4 + vpaddd %ymm11,%ymm0,%ymm4 // add K_00_19 vpaddd %ymm11,%ymm1,%ymm5 - vmovdqu %ymm4,0(%rsp) + vmovdqu %ymm4,0(%rsp) // X[]+K xfer to IALU vpaddd %ymm11,%ymm2,%ymm6 vmovdqu %ymm5,32(%rsp) vpaddd %ymm11,%ymm3,%ymm7 @@ -3867,7 +3867,7 @@ andnl %edx,%ebp,%edi andl %ecx,%ebp xorl %edi,%ebp - jmp .Lalign32_1 + jmp .Lalign32_1 // see "Decoded ICache" in manual .align 32 .Lalign32_1: vpalignr $8,%ymm6,%ymm7,%ymm8 @@ -4246,7 +4246,7 @@ xorl %esi,%ecx addl %r12d,%ebx andl %edi,%ecx - jmp .Lalign32_2 + jmp .Lalign32_2 // see "Decoded ICache" in manual .align 32 .Lalign32_2: vpalignr $8,%ymm6,%ymm7,%ymm8 @@ -4622,38 +4622,38 @@ rorxl $27,%esi,%r12d addl %r12d,%edx leaq 128(%r9),%r13 - leaq 128(%r9),%rdi + leaq 128(%r9),%rdi // borrow %edi cmpq %r10,%r13 - cmovaeq %r9,%r13 + cmovaeq %r9,%r13 // next or previous block - - addl 0(%r8),%edx +// output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + addl 0(%r8),%edx // update context addl 4(%r8),%esi addl 8(%r8),%ebp movl %edx,0(%r8) addl 12(%r8),%ebx movl %esi,4(%r8) - movl %edx,%eax + movl %edx,%eax // A=d addl 16(%r8),%ecx movl %ebp,%r12d movl %ebp,8(%r8) - movl %ebx,%edx - + movl %ebx,%edx // D=b +// xchg %ecx,%ebp # F=c, C=f movl %ebx,12(%r8) - movl %esi,%ebp + movl %esi,%ebp // F=e movl %ecx,16(%r8) - - movl %ecx,%esi - movl %r12d,%ecx - +// mov %ebp,16(%r8) + movl %ecx,%esi // E=c + movl %r12d,%ecx // C=f +// xchg %ebp,%esi # E=c, F=e cmpq %r10,%r9 je .Ldone_avx2 - vmovdqu 64(%r14),%ymm6 - cmpq %r10,%rdi + vmovdqu 64(%r14),%ymm6 // pbswap mask + cmpq %r10,%rdi // borrowed %edi ja .Last_avx2 - vmovdqu -64(%rdi),%xmm0 + vmovdqu -64(%rdi),%xmm0 // low part of %ymm0 vmovdqu -48(%rdi),%xmm1 vmovdqu -32(%rdi),%xmm2 vmovdqu -16(%rdi),%xmm3 @@ -5033,7 +5033,7 @@ xorl %eax,%edx addl %r12d,%ecx andl %edi,%edx - jmp .Lalign32_3 + jmp .Lalign32_3 // see "Decoded ICache" in manual .align 32 .Lalign32_3: vmovdqu %ymm6,64(%rsp) @@ -5388,26 +5388,26 @@ addl %r12d,%edx leaq 128(%rsp),%r13 - - addl 0(%r8),%edx +// output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + addl 0(%r8),%edx // update context addl 4(%r8),%esi addl 8(%r8),%ebp movl %edx,0(%r8) addl 12(%r8),%ebx movl %esi,4(%r8) - movl %edx,%eax + movl %edx,%eax // A=d addl 16(%r8),%ecx movl %ebp,%r12d movl %ebp,8(%r8) - movl %ebx,%edx - + movl %ebx,%edx // D=b +// xchg %ecx,%ebp # F=c, C=f movl %ebx,12(%r8) - movl %esi,%ebp + movl %esi,%ebp // F=e movl %ecx,16(%r8) - - movl %ecx,%esi - movl %r12d,%ecx - +// mov %ebp,16(%r8) + movl %ecx,%esi // E=c + movl %r12d,%ecx // C=f +// xchg %ebp,%esi # E=c, F=e cmpq %r10,%r9 jbe .Loop_avx2 @@ -5433,16 +5433,16 @@ .section .rodata .align 64 K_XX_XX: -.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 -.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 -.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 -.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 -.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc -.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc -.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 -.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 -.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f -.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 // K_00_19 +.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 // K_00_19 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 // K_20_39 +.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 // K_20_39 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc // K_40_59 +.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc // K_40_59 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 // K_60_79 +.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 // K_60_79 +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f // pbswap mask +.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f // pbswap mask .byte 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 .byte 83,72,65,49,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 .align 64
diff --git a/gen/bcm/sha1-x86_64-win.asm b/gen/bcm/sha1-x86_64-win.asm index 128fc37..b59fb5d 100644 --- a/gen/bcm/sha1-x86_64-win.asm +++ b/gen/bcm/sha1-x86_64-win.asm
@@ -41,11 +41,11 @@ push r14 - mov r8,rdi + mov r8,rdi ; reassigned argument sub rsp,72 - mov r9,rsi + mov r9,rsi ; reassigned argument and rsp,-64 - mov r10,rdx + mov r10,rdx ; reassigned argument mov QWORD[64+rsp],rax $L$prologue: @@ -1284,32 +1284,32 @@ $L$prologue_shaext: movdqu xmm0,XMMWORD[rdi] movd xmm1,DWORD[16+rdi] - movdqa xmm3,XMMWORD[((K_XX_XX+160))] + movdqa xmm3,XMMWORD[((K_XX_XX+160))] ; byte-n-word swap movdqu xmm4,XMMWORD[rsi] - pshufd xmm0,xmm0,27 + pshufd xmm0,xmm0,27 ; flip word order movdqu xmm5,XMMWORD[16+rsi] - pshufd xmm1,xmm1,27 + pshufd xmm1,xmm1,27 ; flip word order movdqu xmm6,XMMWORD[32+rsi] pshufb xmm4,xmm3 movdqu xmm7,XMMWORD[48+rsi] pshufb xmm5,xmm3 pshufb xmm6,xmm3 - movdqa xmm9,xmm1 + movdqa xmm9,xmm1 ; offload %xmm1 pshufb xmm7,xmm3 jmp NEAR $L$oop_shaext ALIGN 16 $L$oop_shaext: dec rdx - lea r8,[64+rsi] + lea r8,[64+rsi] ; next input block paddd xmm1,xmm4 cmovne rsi,r8 prefetcht0 [512+rsi] - movdqa xmm8,xmm0 + movdqa xmm8,xmm0 ; offload %xmm0 sha1msg1 xmm4,xmm5 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,0 + sha1rnds4 xmm0,xmm1,0 ; 0-3... sha1nexte xmm2,xmm5 pxor xmm4,xmm6 sha1msg1 xmm5,xmm6 @@ -1322,7 +1322,7 @@ sha1msg2 xmm5,xmm4 sha1msg1 xmm6,xmm7 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,0 + sha1rnds4 xmm0,xmm1,0 ; 0-3... sha1nexte xmm2,xmm7 pxor xmm6,xmm4 sha1msg1 xmm7,xmm4 @@ -1335,7 +1335,7 @@ sha1msg2 xmm7,xmm6 sha1msg1 xmm4,xmm5 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,0 + sha1rnds4 xmm0,xmm1,0 ; 0-3... sha1nexte xmm2,xmm5 pxor xmm4,xmm6 sha1msg1 xmm5,xmm6 @@ -1348,7 +1348,7 @@ sha1msg2 xmm5,xmm4 sha1msg1 xmm6,xmm7 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,1 + sha1rnds4 xmm0,xmm1,1 ; 0-3... sha1nexte xmm2,xmm7 pxor xmm6,xmm4 sha1msg1 xmm7,xmm4 @@ -1361,7 +1361,7 @@ sha1msg2 xmm7,xmm6 sha1msg1 xmm4,xmm5 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,1 + sha1rnds4 xmm0,xmm1,1 ; 0-3... sha1nexte xmm2,xmm5 pxor xmm4,xmm6 sha1msg1 xmm5,xmm6 @@ -1374,7 +1374,7 @@ sha1msg2 xmm5,xmm4 sha1msg1 xmm6,xmm7 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,2 + sha1rnds4 xmm0,xmm1,2 ; 0-3... sha1nexte xmm2,xmm7 pxor xmm6,xmm4 sha1msg1 xmm7,xmm4 @@ -1387,7 +1387,7 @@ sha1msg2 xmm7,xmm6 sha1msg1 xmm4,xmm5 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,2 + sha1rnds4 xmm0,xmm1,2 ; 0-3... sha1nexte xmm2,xmm5 pxor xmm4,xmm6 sha1msg1 xmm5,xmm6 @@ -1400,7 +1400,7 @@ sha1msg2 xmm5,xmm4 sha1msg1 xmm6,xmm7 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,2 + sha1rnds4 xmm0,xmm1,2 ; 0-3... sha1nexte xmm2,xmm7 pxor xmm6,xmm4 sha1msg1 xmm7,xmm4 @@ -1413,30 +1413,30 @@ sha1msg2 xmm7,xmm6 movdqu xmm4,XMMWORD[rsi] movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,3 + sha1rnds4 xmm0,xmm1,3 ; 64-67 sha1nexte xmm2,xmm5 movdqu xmm5,XMMWORD[16+rsi] pshufb xmm4,xmm3 movdqa xmm1,xmm0 - sha1rnds4 xmm0,xmm2,3 + sha1rnds4 xmm0,xmm2,3 ; 68-71 sha1nexte xmm1,xmm6 movdqu xmm6,XMMWORD[32+rsi] pshufb xmm5,xmm3 movdqa xmm2,xmm0 - sha1rnds4 xmm0,xmm1,3 + sha1rnds4 xmm0,xmm1,3 ; 72-75 sha1nexte xmm2,xmm7 movdqu xmm7,XMMWORD[48+rsi] pshufb xmm6,xmm3 movdqa xmm1,xmm0 - sha1rnds4 xmm0,xmm2,3 + sha1rnds4 xmm0,xmm2,3 ; 76-79 sha1nexte xmm1,xmm9 pshufb xmm7,xmm3 paddd xmm0,xmm8 - movdqa xmm9,xmm1 + movdqa xmm9,xmm1 ; offload %xmm1 jnz NEAR $L$oop_shaext @@ -1470,7 +1470,7 @@ _CET_ENDBR - mov r11,rsp + mov r11,rsp ; frame pointer push rbx @@ -1478,7 +1478,7 @@ push r12 - push r13 + push r13 ; redundant, done to share Win64 SE handler push r14 @@ -1491,40 +1491,40 @@ movaps XMMWORD[(-40-16)+r11],xmm11 $L$prologue_ssse3: and rsp,-64 - mov r8,rdi - mov r9,rsi - mov r10,rdx + mov r8,rdi ; reassigned argument + mov r9,rsi ; reassigned argument + mov r10,rdx ; reassigned argument shl r10,6 add r10,r9 lea r14,[((K_XX_XX+64))] - mov eax,DWORD[r8] + mov eax,DWORD[r8] ; load context mov ebx,DWORD[4+r8] mov ecx,DWORD[8+r8] mov edx,DWORD[12+r8] - mov esi,ebx + mov esi,ebx ; magic seed mov ebp,DWORD[16+r8] mov edi,ecx xor edi,edx and esi,edi - movdqa xmm6,XMMWORD[64+r14] - movdqa xmm9,XMMWORD[((-64))+r14] - movdqu xmm0,XMMWORD[r9] + movdqa xmm6,XMMWORD[64+r14] ; pbswap mask + movdqa xmm9,XMMWORD[((-64))+r14] ; K_00_19 + movdqu xmm0,XMMWORD[r9] ; load input to %xmm[0-3] movdqu xmm1,XMMWORD[16+r9] movdqu xmm2,XMMWORD[32+r9] movdqu xmm3,XMMWORD[48+r9] - pshufb xmm0,xmm6 + pshufb xmm0,xmm6 ; byte swap pshufb xmm1,xmm6 pshufb xmm2,xmm6 add r9,64 - paddd xmm0,xmm9 + paddd xmm0,xmm9 ; add K_00_19 pshufb xmm3,xmm6 paddd xmm1,xmm9 paddd xmm2,xmm9 - movdqa XMMWORD[rsp],xmm0 - psubd xmm0,xmm9 + movdqa XMMWORD[rsp],xmm0 ; X[]+K xfer to IALU + psubd xmm0,xmm9 ; restore X[] movdqa XMMWORD[16+rsp],xmm1 psubd xmm1,xmm9 movdqa XMMWORD[32+rsp],xmm2 @@ -2521,14 +2521,14 @@ add eax,edi ror ecx,7 add eax,ebx - add eax,DWORD[r8] + add eax,DWORD[r8] ; update context add esi,DWORD[4+r8] add ecx,DWORD[8+r8] add edx,DWORD[12+r8] mov DWORD[r8],eax add ebp,DWORD[16+r8] mov DWORD[4+r8],esi - mov ebx,esi + mov ebx,esi ; magic seed mov DWORD[8+r8],ecx mov edi,ecx mov DWORD[12+r8],edx @@ -2634,7 +2634,7 @@ add eax,edi ror ecx,7 add eax,ebx - add eax,DWORD[r8] + add eax,DWORD[r8] ; update context add esi,DWORD[4+r8] add ecx,DWORD[8+r8] mov DWORD[r8],eax @@ -2691,7 +2691,7 @@ push r12 - push r13 + push r13 ; redundant, done to share Win64 SE handler push r14 @@ -2705,39 +2705,39 @@ vmovaps XMMWORD[(-40-16)+r11],xmm11 $L$prologue_avx: and rsp,-64 - mov r8,rdi - mov r9,rsi - mov r10,rdx + mov r8,rdi ; reassigned argument + mov r9,rsi ; reassigned argument + mov r10,rdx ; reassigned argument shl r10,6 add r10,r9 lea r14,[((K_XX_XX+64))] - mov eax,DWORD[r8] + mov eax,DWORD[r8] ; load context mov ebx,DWORD[4+r8] mov ecx,DWORD[8+r8] mov edx,DWORD[12+r8] - mov esi,ebx + mov esi,ebx ; magic seed mov ebp,DWORD[16+r8] mov edi,ecx xor edi,edx and esi,edi - vmovdqa xmm6,XMMWORD[64+r14] - vmovdqa xmm11,XMMWORD[((-64))+r14] - vmovdqu xmm0,XMMWORD[r9] + vmovdqa xmm6,XMMWORD[64+r14] ; pbswap mask + vmovdqa xmm11,XMMWORD[((-64))+r14] ; K_00_19 + vmovdqu xmm0,XMMWORD[r9] ; load input to %xmm[0-3] vmovdqu xmm1,XMMWORD[16+r9] vmovdqu xmm2,XMMWORD[32+r9] vmovdqu xmm3,XMMWORD[48+r9] - vpshufb xmm0,xmm0,xmm6 + vpshufb xmm0,xmm0,xmm6 ; byte swap add r9,64 vpshufb xmm1,xmm1,xmm6 vpshufb xmm2,xmm2,xmm6 vpshufb xmm3,xmm3,xmm6 - vpaddd xmm4,xmm0,xmm11 + vpaddd xmm4,xmm0,xmm11 ; add K_00_19 vpaddd xmm5,xmm1,xmm11 vpaddd xmm6,xmm2,xmm11 - vmovdqa XMMWORD[rsp],xmm4 + vmovdqa XMMWORD[rsp],xmm4 ; X[]+K xfer to IALU vmovdqa XMMWORD[16+rsp],xmm5 vmovdqa XMMWORD[32+rsp],xmm6 jmp NEAR $L$oop_avx @@ -3672,14 +3672,14 @@ add eax,edi shrd ecx,ecx,7 add eax,ebx - add eax,DWORD[r8] + add eax,DWORD[r8] ; update context add esi,DWORD[4+r8] add ecx,DWORD[8+r8] add edx,DWORD[12+r8] mov DWORD[r8],eax add ebp,DWORD[16+r8] mov DWORD[4+r8],esi - mov ebx,esi + mov ebx,esi ; magic seed mov DWORD[8+r8],ecx mov edi,ecx mov DWORD[12+r8],edx @@ -3787,7 +3787,7 @@ add eax,ebx vzeroupper - add eax,DWORD[r8] + add eax,DWORD[r8] ; update context add esi,DWORD[4+r8] add ecx,DWORD[8+r8] mov DWORD[r8],eax @@ -3857,9 +3857,9 @@ vmovaps XMMWORD[(-40-32)+r11],xmm10 vmovaps XMMWORD[(-40-16)+r11],xmm11 $L$prologue_avx2: - mov r8,rdi - mov r9,rsi - mov r10,rdx + mov r8,rdi ; reassigned argument + mov r9,rsi ; reassigned argument + mov r10,rdx ; reassigned argument lea rsp,[((-640))+rsp] shl r10,6 @@ -3868,14 +3868,14 @@ add r10,r9 lea r14,[((K_XX_XX+64))] - mov eax,DWORD[r8] + mov eax,DWORD[r8] ; load context cmp r13,r10 - cmovae r13,r9 + cmovae r13,r9 ; next or same block mov ebp,DWORD[4+r8] mov ecx,DWORD[8+r8] mov edx,DWORD[12+r8] mov esi,DWORD[16+r8] - vmovdqu ymm6,YMMWORD[64+r14] + vmovdqu ymm6,YMMWORD[64+r14] ; pbswap mask vmovdqu xmm0,XMMWORD[r9] vmovdqu xmm1,XMMWORD[16+r9] @@ -3889,12 +3889,12 @@ vpshufb ymm1,ymm1,ymm6 vinserti128 ymm3,ymm3,XMMWORD[48+r13],1 vpshufb ymm2,ymm2,ymm6 - vmovdqu ymm11,YMMWORD[((-64))+r14] + vmovdqu ymm11,YMMWORD[((-64))+r14] ; K_00_19 vpshufb ymm3,ymm3,ymm6 - vpaddd ymm4,ymm0,ymm11 + vpaddd ymm4,ymm0,ymm11 ; add K_00_19 vpaddd ymm5,ymm1,ymm11 - vmovdqu YMMWORD[rsp],ymm4 + vmovdqu YMMWORD[rsp],ymm4 ; X[]+K xfer to IALU vpaddd ymm6,ymm2,ymm11 vmovdqu YMMWORD[32+rsp],ymm5 vpaddd ymm7,ymm3,ymm11 @@ -3969,7 +3969,7 @@ andn edi,ebp,edx and ebp,ecx xor ebp,edi - jmp NEAR $L$align32_1 + jmp NEAR $L$align32_1 ; see "Decoded ICache" in manual ALIGN 32 $L$align32_1: vpalignr ymm8,ymm7,ymm6,8 @@ -4348,7 +4348,7 @@ xor ecx,esi add ebx,r12d and ecx,edi - jmp NEAR $L$align32_2 + jmp NEAR $L$align32_2 ; see "Decoded ICache" in manual ALIGN 32 $L$align32_2: vpalignr ymm8,ymm7,ymm6,8 @@ -4724,38 +4724,38 @@ rorx r12d,esi,27 add edx,r12d lea r13,[128+r9] - lea rdi,[128+r9] + lea rdi,[128+r9] ; borrow %edi cmp r13,r10 - cmovae r13,r9 + cmovae r13,r9 ; next or previous block - - add edx,DWORD[r8] +; output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + add edx,DWORD[r8] ; update context add esi,DWORD[4+r8] add ebp,DWORD[8+r8] mov DWORD[r8],edx add ebx,DWORD[12+r8] mov DWORD[4+r8],esi - mov eax,edx + mov eax,edx ; A=d add ecx,DWORD[16+r8] mov r12d,ebp mov DWORD[8+r8],ebp - mov edx,ebx - + mov edx,ebx ; D=b +; xchg %ecx,%ebp # F=c, C=f mov DWORD[12+r8],ebx - mov ebp,esi + mov ebp,esi ; F=e mov DWORD[16+r8],ecx - - mov esi,ecx - mov ecx,r12d - +; mov %ebp,16(%r8) + mov esi,ecx ; E=c + mov ecx,r12d ; C=f +; xchg %ebp,%esi # E=c, F=e cmp r9,r10 je NEAR $L$done_avx2 - vmovdqu ymm6,YMMWORD[64+r14] - cmp rdi,r10 + vmovdqu ymm6,YMMWORD[64+r14] ; pbswap mask + cmp rdi,r10 ; borrowed %edi ja NEAR $L$ast_avx2 - vmovdqu xmm0,XMMWORD[((-64))+rdi] + vmovdqu xmm0,XMMWORD[((-64))+rdi] ; low part of %ymm0 vmovdqu xmm1,XMMWORD[((-48))+rdi] vmovdqu xmm2,XMMWORD[((-32))+rdi] vmovdqu xmm3,XMMWORD[((-16))+rdi] @@ -5135,7 +5135,7 @@ xor edx,eax add ecx,r12d and edx,edi - jmp NEAR $L$align32_3 + jmp NEAR $L$align32_3 ; see "Decoded ICache" in manual ALIGN 32 $L$align32_3: vmovdqu YMMWORD[64+rsp],ymm6 @@ -5490,26 +5490,26 @@ add edx,r12d lea r13,[128+rsp] - - add edx,DWORD[r8] +; output is d-e-[a]-f-b-c => A=d,F=e,C=f,D=b,E=c + add edx,DWORD[r8] ; update context add esi,DWORD[4+r8] add ebp,DWORD[8+r8] mov DWORD[r8],edx add ebx,DWORD[12+r8] mov DWORD[4+r8],esi - mov eax,edx + mov eax,edx ; A=d add ecx,DWORD[16+r8] mov r12d,ebp mov DWORD[8+r8],ebp - mov edx,ebx - + mov edx,ebx ; D=b +; xchg %ecx,%ebp # F=c, C=f mov DWORD[12+r8],ebx - mov ebp,esi + mov ebp,esi ; F=e mov DWORD[16+r8],ecx - - mov esi,ecx - mov ecx,r12d - +; mov %ebp,16(%r8) + mov esi,ecx ; E=c + mov ecx,r12d ; C=f +; xchg %ebp,%esi # E=c, F=e cmp r9,r10 jbe NEAR $L$oop_avx2 @@ -5543,16 +5543,16 @@ section .rdata rdata align=8 ALIGN 64 K_XX_XX: - DD 0x5a827999,0x5a827999,0x5a827999,0x5a827999 - DD 0x5a827999,0x5a827999,0x5a827999,0x5a827999 - DD 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 - DD 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 - DD 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc - DD 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc - DD 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 - DD 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 - DD 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f - DD 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f + DD 0x5a827999,0x5a827999,0x5a827999,0x5a827999 ; K_00_19 + DD 0x5a827999,0x5a827999,0x5a827999,0x5a827999 ; K_00_19 + DD 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 ; K_20_39 + DD 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 ; K_20_39 + DD 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc ; K_40_59 + DD 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc ; K_40_59 + DD 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 ; K_60_79 + DD 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 ; K_60_79 + DD 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f ; pbswap mask + DD 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f ; pbswap mask DB 0xf,0xe,0xd,0xc,0xb,0xa,0x9,0x8,0x7,0x6,0x5,0x4,0x3,0x2,0x1,0x0 DB 83,72,65,49,32,98,108,111,99,107,32,116,114,97,110,115 DB 102,111,114,109,32,102,111,114,32,120,56,54,95,54,52,44 @@ -5577,31 +5577,31 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip lea r10,[$L$prologue] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<.Lprologue jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp lea r10,[$L$epilogue] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip>=.Lepilogue jae NEAR $L$common_seh_tail - mov rax,QWORD[64+rax] + mov rax,QWORD[64+rax] ; pull saved stack pointer mov rbx,QWORD[((-8))+rax] mov rbp,QWORD[((-16))+rax] mov r12,QWORD[((-24))+rax] mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 jmp NEAR $L$common_seh_tail @@ -5619,21 +5619,21 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip lea r10,[$L$prologue_shaext] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<.Lprologue jb NEAR $L$common_seh_tail lea r10,[$L$epilogue_shaext] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip>=.Lepilogue jae NEAR $L$common_seh_tail lea rsi,[((-8-64))+rax] - lea rdi,[512+r8] + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,8 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq jmp NEAR $L$common_seh_tail @@ -5651,67 +5651,67 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[208+r8] + mov rax,QWORD[208+r8] ; pull context->R11 - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail lea rsi,[((-40-96))+rax] - lea rdi,[512+r8] + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,12 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq mov rbx,QWORD[((-8))+rax] mov rbp,QWORD[((-16))+rax] mov r12,QWORD[((-24))+rax] mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -5753,15 +5753,15 @@ $L$SEH_info_sha1_block_data_order_ssse3: DB 9,0,0,0 DD ssse3_handler wrt ..imagebase - DD $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase + DD $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase ; HandlerData[] $L$SEH_info_sha1_block_data_order_avx: DB 9,0,0,0 DD ssse3_handler wrt ..imagebase - DD $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase + DD $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase ; HandlerData[] $L$SEH_info_sha1_block_data_order_avx2: DB 9,0,0,0 DD ssse3_handler wrt ..imagebase - DD $L$prologue_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase + DD $L$prologue_avx2 wrt ..imagebase,$L$epilogue_avx2 wrt ..imagebase ; HandlerData[] %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/bcm/sha256-x86_64-apple.S b/gen/bcm/sha256-x86_64-apple.S index 814dd25..9400bc5 100644 --- a/gen/bcm/sha256-x86_64-apple.S +++ b/gen/bcm/sha256-x86_64-apple.S
@@ -13,7 +13,7 @@ _sha256_block_data_order_nohw: _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp pushq %rbx @@ -27,14 +27,14 @@ pushq %r15 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $64+32,%rsp - leaq (%rsi,%rdx,4),%rdx - andq $-64,%rsp - movq %rdi,64+0(%rsp) - movq %rsi,64+8(%rsp) - movq %rdx,64+16(%rsp) - movq %rax,88(%rsp) + leaq (%rsi,%rdx,4),%rdx // inp+num*16*4 + andq $-64,%rsp // align stack frame + movq %rdi,64+0(%rsp) // save ctx, 1st arg + movq %rsi,64+8(%rsp) // save inp, 2nd arh + movq %rdx,64+16(%rsp) // save end pointer, "3rd" arg + movq %rax,88(%rsp) // save copy of %rsp L$prologue: @@ -52,7 +52,7 @@ L$loop: movl %ebx,%edi leaq K256(%rip),%rbp - xorl %ecx,%edi + xorl %ecx,%edi // magic movl 0(%rsi),%r12d movl %r8d,%r13d movl %eax,%r14d @@ -62,38 +62,38 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,0(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r11d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r11d // h+=Sigma0(a) movl 4(%rsi),%r12d movl %edx,%r13d movl %r11d,%r14d @@ -103,38 +103,38 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,4(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r10d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r10d // h+=Sigma0(a) movl 8(%rsi),%r12d movl %ecx,%r13d movl %r10d,%r14d @@ -144,38 +144,38 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,8(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r9d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r9d // h+=Sigma0(a) movl 12(%rsi),%r12d movl %ebx,%r13d movl %r9d,%r14d @@ -185,38 +185,38 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,12(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp - addl %r14d,%r8d + leaq 20(%rbp),%rbp // round++ + addl %r14d,%r8d // h+=Sigma0(a) movl 16(%rsi),%r12d movl %eax,%r13d movl %r8d,%r14d @@ -226,38 +226,38 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,16(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%edx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%edx // h+=Sigma0(a) movl 20(%rsi),%r12d movl %r11d,%r13d movl %edx,%r14d @@ -267,38 +267,38 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,20(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ecx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ecx // h+=Sigma0(a) movl 24(%rsi),%r12d movl %r10d,%r13d movl %ecx,%r14d @@ -308,38 +308,38 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,24(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ebx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ebx // h+=Sigma0(a) movl 28(%rsi),%r12d movl %r9d,%r13d movl %ebx,%r14d @@ -349,38 +349,38 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,28(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp - addl %r14d,%eax + leaq 20(%rbp),%rbp // round++ + addl %r14d,%eax // h+=Sigma0(a) movl 32(%rsi),%r12d movl %r8d,%r13d movl %eax,%r14d @@ -390,38 +390,38 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,32(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r11d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r11d // h+=Sigma0(a) movl 36(%rsi),%r12d movl %edx,%r13d movl %r11d,%r14d @@ -431,38 +431,38 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,36(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r10d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r10d // h+=Sigma0(a) movl 40(%rsi),%r12d movl %ecx,%r13d movl %r10d,%r14d @@ -472,38 +472,38 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,40(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r9d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r9d // h+=Sigma0(a) movl 44(%rsi),%r12d movl %ebx,%r13d movl %r9d,%r14d @@ -513,38 +513,38 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,44(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp - addl %r14d,%r8d + leaq 20(%rbp),%rbp // round++ + addl %r14d,%r8d // h+=Sigma0(a) movl 48(%rsi),%r12d movl %eax,%r13d movl %r8d,%r14d @@ -554,38 +554,38 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,48(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%edx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%edx // h+=Sigma0(a) movl 52(%rsi),%r12d movl %r11d,%r13d movl %edx,%r14d @@ -595,38 +595,38 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,52(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ecx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ecx // h+=Sigma0(a) movl 56(%rsi),%r12d movl %r10d,%r13d movl %ecx,%r14d @@ -636,38 +636,38 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,56(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ebx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ebx // h+=Sigma0(a) movl 60(%rsi),%r12d movl %r9d,%r13d movl %ebx,%r14d @@ -677,37 +677,37 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,60(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ jmp L$rounds_16_xx .p2align 4 L$rounds_16_xx: @@ -716,7 +716,7 @@ movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%eax + addl %r14d,%eax // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -727,8 +727,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 36(%rsp),%r12d addl 0(%rsp),%r12d @@ -740,43 +740,43 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,0(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 8(%rsp),%r13d movl 60(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r11d + addl %r14d,%r11d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -787,8 +787,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 40(%rsp),%r12d addl 4(%rsp),%r12d @@ -800,43 +800,43 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,4(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 12(%rsp),%r13d movl 0(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r10d + addl %r14d,%r10d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -847,8 +847,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 44(%rsp),%r12d addl 8(%rsp),%r12d @@ -860,43 +860,43 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,8(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 16(%rsp),%r13d movl 4(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r9d + addl %r14d,%r9d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -907,8 +907,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 48(%rsp),%r12d addl 12(%rsp),%r12d @@ -920,43 +920,43 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,12(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ movl 20(%rsp),%r13d movl 8(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r8d + addl %r14d,%r8d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -967,8 +967,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 52(%rsp),%r12d addl 16(%rsp),%r12d @@ -980,43 +980,43 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,16(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 24(%rsp),%r13d movl 12(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%edx + addl %r14d,%edx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1027,8 +1027,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 56(%rsp),%r12d addl 20(%rsp),%r12d @@ -1040,43 +1040,43 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,20(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 28(%rsp),%r13d movl 16(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ecx + addl %r14d,%ecx // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1087,8 +1087,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 60(%rsp),%r12d addl 24(%rsp),%r12d @@ -1100,43 +1100,43 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,24(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 32(%rsp),%r13d movl 20(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ebx + addl %r14d,%ebx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1147,8 +1147,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 0(%rsp),%r12d addl 28(%rsp),%r12d @@ -1160,43 +1160,43 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,28(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ movl 36(%rsp),%r13d movl 24(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%eax + addl %r14d,%eax // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1207,8 +1207,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 4(%rsp),%r12d addl 32(%rsp),%r12d @@ -1220,43 +1220,43 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,32(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 40(%rsp),%r13d movl 28(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r11d + addl %r14d,%r11d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1267,8 +1267,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 8(%rsp),%r12d addl 36(%rsp),%r12d @@ -1280,43 +1280,43 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,36(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 44(%rsp),%r13d movl 32(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r10d + addl %r14d,%r10d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1327,8 +1327,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 12(%rsp),%r12d addl 40(%rsp),%r12d @@ -1340,43 +1340,43 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,40(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 48(%rsp),%r13d movl 36(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r9d + addl %r14d,%r9d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1387,8 +1387,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 16(%rsp),%r12d addl 44(%rsp),%r12d @@ -1400,43 +1400,43 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,44(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ movl 52(%rsp),%r13d movl 40(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r8d + addl %r14d,%r8d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1447,8 +1447,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 20(%rsp),%r12d addl 48(%rsp),%r12d @@ -1460,43 +1460,43 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,48(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 56(%rsp),%r13d movl 44(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%edx + addl %r14d,%edx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1507,8 +1507,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 24(%rsp),%r12d addl 52(%rsp),%r12d @@ -1520,43 +1520,43 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,52(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 60(%rsp),%r13d movl 48(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ecx + addl %r14d,%ecx // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1567,8 +1567,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 28(%rsp),%r12d addl 56(%rsp),%r12d @@ -1580,43 +1580,43 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,56(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 0(%rsp),%r13d movl 52(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ebx + addl %r14d,%ebx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1627,8 +1627,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 32(%rsp),%r12d addl 60(%rsp),%r12d @@ -1640,42 +1640,42 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,60(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ cmpb $0,3(%rbp) jnz L$rounds_16_xx movq 64+0(%rsp),%rdi - addl %r14d,%eax + addl %r14d,%eax // modulo-scheduled h+=Sigma0(a) leaq 64(%rsi),%rsi addl 0(%rdi),%eax @@ -1773,16 +1773,16 @@ _CET_ENDBR leaq K256+128(%rip),%rcx - movdqu (%rdi),%xmm1 - movdqu 16(%rdi),%xmm2 - movdqa 512-128(%rcx),%xmm7 + movdqu (%rdi),%xmm1 // DCBA + movdqu 16(%rdi),%xmm2 // HGFE + movdqa 512-128(%rcx),%xmm7 // byte swap mask - pshufd $0x1b,%xmm1,%xmm0 - pshufd $0xb1,%xmm1,%xmm1 - pshufd $0x1b,%xmm2,%xmm2 - movdqa %xmm7,%xmm8 - palignr $8,%xmm2,%xmm1 - punpcklqdq %xmm0,%xmm2 + pshufd $0x1b,%xmm1,%xmm0 // ABCD + pshufd $0xb1,%xmm1,%xmm1 // CDAB + pshufd $0x1b,%xmm2,%xmm2 // EFGH + movdqa %xmm7,%xmm8 // offload + palignr $8,%xmm2,%xmm1 // ABEF + punpcklqdq %xmm0,%xmm2 // CDGH jmp L$oop_shaext .p2align 4 @@ -1796,17 +1796,17 @@ movdqa 0-128(%rcx),%xmm0 paddd %xmm3,%xmm0 pshufb %xmm7,%xmm4 - movdqa %xmm2,%xmm10 - sha256rnds2 %xmm1,%xmm2 + movdqa %xmm2,%xmm10 // offload + sha256rnds2 %xmm1,%xmm2 // 0-3 pshufd $0x0e,%xmm0,%xmm0 nop - movdqa %xmm1,%xmm9 + movdqa %xmm1,%xmm9 // offload sha256rnds2 %xmm2,%xmm1 movdqa 32-128(%rcx),%xmm0 paddd %xmm4,%xmm0 pshufb %xmm7,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 4-7 pshufd $0x0e,%xmm0,%xmm0 leaq 64(%rsi),%rsi sha256msg1 %xmm4,%xmm3 @@ -1815,7 +1815,7 @@ movdqa 64-128(%rcx),%xmm0 paddd %xmm5,%xmm0 pshufb %xmm7,%xmm6 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 8-11 pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm6,%xmm7 palignr $4,%xmm5,%xmm7 @@ -1827,7 +1827,7 @@ movdqa 96-128(%rcx),%xmm0 paddd %xmm6,%xmm0 sha256msg2 %xmm6,%xmm3 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 12-15 pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm3,%xmm7 palignr $4,%xmm6,%xmm7 @@ -1838,7 +1838,7 @@ movdqa 128-128(%rcx),%xmm0 paddd %xmm3,%xmm0 sha256msg2 %xmm3,%xmm4 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm4,%xmm7 palignr $4,%xmm3,%xmm7 @@ -1849,7 +1849,7 @@ movdqa 160-128(%rcx),%xmm0 paddd %xmm4,%xmm0 sha256msg2 %xmm4,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm5,%xmm7 palignr $4,%xmm4,%xmm7 @@ -1860,7 +1860,7 @@ movdqa 192-128(%rcx),%xmm0 paddd %xmm5,%xmm0 sha256msg2 %xmm5,%xmm6 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm6,%xmm7 palignr $4,%xmm5,%xmm7 @@ -1871,7 +1871,7 @@ movdqa 224-128(%rcx),%xmm0 paddd %xmm6,%xmm0 sha256msg2 %xmm6,%xmm3 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm3,%xmm7 palignr $4,%xmm6,%xmm7 @@ -1882,7 +1882,7 @@ movdqa 256-128(%rcx),%xmm0 paddd %xmm3,%xmm0 sha256msg2 %xmm3,%xmm4 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm4,%xmm7 palignr $4,%xmm3,%xmm7 @@ -1893,7 +1893,7 @@ movdqa 288-128(%rcx),%xmm0 paddd %xmm4,%xmm0 sha256msg2 %xmm4,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm5,%xmm7 palignr $4,%xmm4,%xmm7 @@ -1904,7 +1904,7 @@ movdqa 320-128(%rcx),%xmm0 paddd %xmm5,%xmm0 sha256msg2 %xmm5,%xmm6 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm6,%xmm7 palignr $4,%xmm5,%xmm7 @@ -1915,7 +1915,7 @@ movdqa 352-128(%rcx),%xmm0 paddd %xmm6,%xmm0 sha256msg2 %xmm6,%xmm3 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm3,%xmm7 palignr $4,%xmm6,%xmm7 @@ -1926,7 +1926,7 @@ movdqa 384-128(%rcx),%xmm0 paddd %xmm3,%xmm0 sha256msg2 %xmm3,%xmm4 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm4,%xmm7 palignr $4,%xmm3,%xmm7 @@ -1937,7 +1937,7 @@ movdqa 416-128(%rcx),%xmm0 paddd %xmm4,%xmm0 sha256msg2 %xmm4,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 52-55 pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm5,%xmm7 palignr $4,%xmm4,%xmm7 @@ -1946,7 +1946,7 @@ movdqa 448-128(%rcx),%xmm0 paddd %xmm5,%xmm0 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 56-59 pshufd $0x0e,%xmm0,%xmm0 sha256msg2 %xmm5,%xmm6 movdqa %xmm8,%xmm7 @@ -1955,7 +1955,7 @@ movdqa 480-128(%rcx),%xmm0 paddd %xmm6,%xmm0 nop - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 60-63 pshufd $0x0e,%xmm0,%xmm0 decq %rdx nop @@ -1965,11 +1965,11 @@ paddd %xmm9,%xmm1 jnz L$oop_shaext - pshufd $0xb1,%xmm2,%xmm2 - pshufd $0x1b,%xmm1,%xmm7 - pshufd $0xb1,%xmm1,%xmm1 - punpckhqdq %xmm2,%xmm1 - palignr $8,%xmm7,%xmm2 + pshufd $0xb1,%xmm2,%xmm2 // DCHG + pshufd $0x1b,%xmm1,%xmm7 // FEBA + pshufd $0xb1,%xmm1,%xmm1 // BAFE + punpckhqdq %xmm2,%xmm1 // DCBA + palignr $8,%xmm7,%xmm2 // HGFE movdqu %xmm1,(%rdi) movdqu %xmm2,16(%rdi) @@ -1984,7 +1984,7 @@ _sha256_block_data_order_ssse3: _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp pushq %rbx @@ -1998,14 +1998,14 @@ pushq %r15 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $96,%rsp - leaq (%rsi,%rdx,4),%rdx - andq $-64,%rsp - movq %rdi,64+0(%rsp) - movq %rsi,64+8(%rsp) - movq %rdx,64+16(%rsp) - movq %rax,88(%rsp) + leaq (%rsi,%rdx,4),%rdx // inp+num*16*4 + andq $-64,%rsp // align stack frame + movq %rdi,64+0(%rsp) // save ctx, 1st arg + movq %rsi,64+8(%rsp) // save inp, 2nd arh + movq %rdx,64+16(%rsp) // save end pointer, "3rd" arg + movq %rax,88(%rsp) // save copy of %rsp L$prologue_ssse3: @@ -2017,8 +2017,8 @@ movl 20(%rdi),%r9d movl 24(%rdi),%r10d movl 28(%rdi),%r11d - - +// movdqa K256+512+32(%rip),%xmm8 +// movdqa K256+512+64(%rip),%xmm9 jmp L$loop_ssse3 .p2align 4 L$loop_ssse3: @@ -2045,14 +2045,14 @@ movdqa %xmm5,16(%rsp) movl %ebx,%edi movdqa %xmm6,32(%rsp) - xorl %ecx,%edi + xorl %ecx,%edi // magic movdqa %xmm7,48(%rsp) movl %r8d,%r13d jmp L$ssse3_00_47 .p2align 4 L$ssse3_00_47: - subq $-128,%rbp + subq $-128,%rbp // size optimization rorl $14,%r13d movdqa %xmm1,%xmm4 movl %r14d,%eax @@ -3100,7 +3100,7 @@ _sha256_block_data_order_avx: _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp pushq %rbx @@ -3114,14 +3114,14 @@ pushq %r15 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $96,%rsp - leaq (%rsi,%rdx,4),%rdx - andq $-64,%rsp - movq %rdi,64+0(%rsp) - movq %rsi,64+8(%rsp) - movq %rdx,64+16(%rsp) - movq %rax,88(%rsp) + leaq (%rsi,%rdx,4),%rdx // inp+num*16*4 + andq $-64,%rsp // align stack frame + movq %rdi,64+0(%rsp) // save ctx, 1st arg + movq %rsi,64+8(%rsp) // save inp, 2nd arh + movq %rdx,64+16(%rsp) // save end pointer, "3rd" arg + movq %rax,88(%rsp) // save copy of %rsp L$prologue_avx: @@ -3158,14 +3158,14 @@ vmovdqa %xmm5,16(%rsp) movl %ebx,%edi vmovdqa %xmm6,32(%rsp) - xorl %ecx,%edi + xorl %ecx,%edi // magic vmovdqa %xmm7,48(%rsp) movl %r8d,%r13d jmp L$avx_00_47 .p2align 4 L$avx_00_47: - subq $-128,%rbp + subq $-128,%rbp // size optimization vpalignr $4,%xmm0,%xmm1,%xmm4 shrdl $14,%r13d,%r13d movl %r14d,%eax
diff --git a/gen/bcm/sha256-x86_64-linux.S b/gen/bcm/sha256-x86_64-linux.S index 938f531..6bd2b14 100644 --- a/gen/bcm/sha256-x86_64-linux.S +++ b/gen/bcm/sha256-x86_64-linux.S
@@ -13,7 +13,7 @@ sha256_block_data_order_nohw: .cfi_startproc _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp .cfi_def_cfa_register %rax pushq %rbx .cfi_offset %rbx,-16 @@ -27,14 +27,14 @@ .cfi_offset %r14,-48 pushq %r15 .cfi_offset %r15,-56 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $64+32,%rsp - leaq (%rsi,%rdx,4),%rdx - andq $-64,%rsp - movq %rdi,64+0(%rsp) - movq %rsi,64+8(%rsp) - movq %rdx,64+16(%rsp) - movq %rax,88(%rsp) + leaq (%rsi,%rdx,4),%rdx // inp+num*16*4 + andq $-64,%rsp // align stack frame + movq %rdi,64+0(%rsp) // save ctx, 1st arg + movq %rsi,64+8(%rsp) // save inp, 2nd arh + movq %rdx,64+16(%rsp) // save end pointer, "3rd" arg + movq %rax,88(%rsp) // save copy of %rsp .cfi_escape 0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08 .Lprologue: @@ -52,7 +52,7 @@ .Lloop: movl %ebx,%edi leaq K256(%rip),%rbp - xorl %ecx,%edi + xorl %ecx,%edi // magic movl 0(%rsi),%r12d movl %r8d,%r13d movl %eax,%r14d @@ -62,38 +62,38 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,0(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r11d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r11d // h+=Sigma0(a) movl 4(%rsi),%r12d movl %edx,%r13d movl %r11d,%r14d @@ -103,38 +103,38 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,4(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r10d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r10d // h+=Sigma0(a) movl 8(%rsi),%r12d movl %ecx,%r13d movl %r10d,%r14d @@ -144,38 +144,38 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,8(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r9d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r9d // h+=Sigma0(a) movl 12(%rsi),%r12d movl %ebx,%r13d movl %r9d,%r14d @@ -185,38 +185,38 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,12(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp - addl %r14d,%r8d + leaq 20(%rbp),%rbp // round++ + addl %r14d,%r8d // h+=Sigma0(a) movl 16(%rsi),%r12d movl %eax,%r13d movl %r8d,%r14d @@ -226,38 +226,38 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,16(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%edx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%edx // h+=Sigma0(a) movl 20(%rsi),%r12d movl %r11d,%r13d movl %edx,%r14d @@ -267,38 +267,38 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,20(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ecx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ecx // h+=Sigma0(a) movl 24(%rsi),%r12d movl %r10d,%r13d movl %ecx,%r14d @@ -308,38 +308,38 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,24(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ebx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ebx // h+=Sigma0(a) movl 28(%rsi),%r12d movl %r9d,%r13d movl %ebx,%r14d @@ -349,38 +349,38 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,28(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp - addl %r14d,%eax + leaq 20(%rbp),%rbp // round++ + addl %r14d,%eax // h+=Sigma0(a) movl 32(%rsi),%r12d movl %r8d,%r13d movl %eax,%r14d @@ -390,38 +390,38 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,32(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r11d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r11d // h+=Sigma0(a) movl 36(%rsi),%r12d movl %edx,%r13d movl %r11d,%r14d @@ -431,38 +431,38 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,36(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r10d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r10d // h+=Sigma0(a) movl 40(%rsi),%r12d movl %ecx,%r13d movl %r10d,%r14d @@ -472,38 +472,38 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,40(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%r9d + leaq 4(%rbp),%rbp // round++ + addl %r14d,%r9d // h+=Sigma0(a) movl 44(%rsi),%r12d movl %ebx,%r13d movl %r9d,%r14d @@ -513,38 +513,38 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,44(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp - addl %r14d,%r8d + leaq 20(%rbp),%rbp // round++ + addl %r14d,%r8d // h+=Sigma0(a) movl 48(%rsi),%r12d movl %eax,%r13d movl %r8d,%r14d @@ -554,38 +554,38 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,48(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%edx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%edx // h+=Sigma0(a) movl 52(%rsi),%r12d movl %r11d,%r13d movl %edx,%r14d @@ -595,38 +595,38 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,52(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ecx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ecx // h+=Sigma0(a) movl 56(%rsi),%r12d movl %r10d,%r13d movl %ecx,%r14d @@ -636,38 +636,38 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,56(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp - addl %r14d,%ebx + leaq 4(%rbp),%rbp // round++ + addl %r14d,%ebx // h+=Sigma0(a) movl 60(%rsi),%r12d movl %r9d,%r13d movl %ebx,%r14d @@ -677,37 +677,37 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,60(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ jmp .Lrounds_16_xx .align 16 .Lrounds_16_xx: @@ -716,7 +716,7 @@ movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%eax + addl %r14d,%eax // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -727,8 +727,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 36(%rsp),%r12d addl 0(%rsp),%r12d @@ -740,43 +740,43 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,0(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 8(%rsp),%r13d movl 60(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r11d + addl %r14d,%r11d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -787,8 +787,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 40(%rsp),%r12d addl 4(%rsp),%r12d @@ -800,43 +800,43 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,4(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 12(%rsp),%r13d movl 0(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r10d + addl %r14d,%r10d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -847,8 +847,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 44(%rsp),%r12d addl 8(%rsp),%r12d @@ -860,43 +860,43 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,8(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 16(%rsp),%r13d movl 4(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r9d + addl %r14d,%r9d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -907,8 +907,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 48(%rsp),%r12d addl 12(%rsp),%r12d @@ -920,43 +920,43 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,12(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ movl 20(%rsp),%r13d movl 8(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r8d + addl %r14d,%r8d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -967,8 +967,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 52(%rsp),%r12d addl 16(%rsp),%r12d @@ -980,43 +980,43 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,16(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 24(%rsp),%r13d movl 12(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%edx + addl %r14d,%edx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1027,8 +1027,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 56(%rsp),%r12d addl 20(%rsp),%r12d @@ -1040,43 +1040,43 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,20(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 28(%rsp),%r13d movl 16(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ecx + addl %r14d,%ecx // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1087,8 +1087,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 60(%rsp),%r12d addl 24(%rsp),%r12d @@ -1100,43 +1100,43 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,24(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 32(%rsp),%r13d movl 20(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ebx + addl %r14d,%ebx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1147,8 +1147,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 0(%rsp),%r12d addl 28(%rsp),%r12d @@ -1160,43 +1160,43 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,28(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ movl 36(%rsp),%r13d movl 24(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%eax + addl %r14d,%eax // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1207,8 +1207,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 4(%rsp),%r12d addl 32(%rsp),%r12d @@ -1220,43 +1220,43 @@ xorl %r8d,%r13d rorl $9,%r14d - xorl %r10d,%r15d + xorl %r10d,%r15d // f^g movl %r12d,32(%rsp) xorl %eax,%r14d - andl %r8d,%r15d + andl %r8d,%r15d // (f^g)&e rorl $5,%r13d - addl %r11d,%r12d - xorl %r10d,%r15d + addl %r11d,%r12d // T1+=h + xorl %r10d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r8d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %eax,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %eax,%r14d - xorl %ebx,%r15d - rorl $6,%r13d + xorl %ebx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ebx,%r11d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r11d - addl %r12d,%edx - addl %r12d,%r11d + xorl %edi,%r11d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%edx // d+=T1 + addl %r12d,%r11d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 40(%rsp),%r13d movl 28(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r11d + addl %r14d,%r11d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1267,8 +1267,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 8(%rsp),%r12d addl 36(%rsp),%r12d @@ -1280,43 +1280,43 @@ xorl %edx,%r13d rorl $9,%r14d - xorl %r9d,%edi + xorl %r9d,%edi // f^g movl %r12d,36(%rsp) xorl %r11d,%r14d - andl %edx,%edi + andl %edx,%edi // (f^g)&e rorl $5,%r13d - addl %r10d,%r12d - xorl %r9d,%edi + addl %r10d,%r12d // T1+=h + xorl %r9d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %edx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r11d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r11d,%r14d - xorl %eax,%edi - rorl $6,%r13d + xorl %eax,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %eax,%r10d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r10d - addl %r12d,%ecx - addl %r12d,%r10d + xorl %r15d,%r10d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ecx // d+=T1 + addl %r12d,%r10d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 44(%rsp),%r13d movl 32(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r10d + addl %r14d,%r10d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1327,8 +1327,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 12(%rsp),%r12d addl 40(%rsp),%r12d @@ -1340,43 +1340,43 @@ xorl %ecx,%r13d rorl $9,%r14d - xorl %r8d,%r15d + xorl %r8d,%r15d // f^g movl %r12d,40(%rsp) xorl %r10d,%r14d - andl %ecx,%r15d + andl %ecx,%r15d // (f^g)&e rorl $5,%r13d - addl %r9d,%r12d - xorl %r8d,%r15d + addl %r9d,%r12d // T1+=h + xorl %r8d,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ecx,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r10d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r10d,%r14d - xorl %r11d,%r15d - rorl $6,%r13d + xorl %r11d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r11d,%r9d andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%r9d - addl %r12d,%ebx - addl %r12d,%r9d + xorl %edi,%r9d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%ebx // d+=T1 + addl %r12d,%r9d // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 48(%rsp),%r13d movl 36(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r9d + addl %r14d,%r9d // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1387,8 +1387,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 16(%rsp),%r12d addl 44(%rsp),%r12d @@ -1400,43 +1400,43 @@ xorl %ebx,%r13d rorl $9,%r14d - xorl %edx,%edi + xorl %edx,%edi // f^g movl %r12d,44(%rsp) xorl %r9d,%r14d - andl %ebx,%edi + andl %ebx,%edi // (f^g)&e rorl $5,%r13d - addl %r8d,%r12d - xorl %edx,%edi + addl %r8d,%r12d // T1+=h + xorl %edx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %ebx,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %r9d,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r9d,%r14d - xorl %r10d,%edi - rorl $6,%r13d + xorl %r10d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r10d,%r8d andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%r8d - addl %r12d,%eax - addl %r12d,%r8d + xorl %r15d,%r8d // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%eax // d+=T1 + addl %r12d,%r8d // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ movl 52(%rsp),%r13d movl 40(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%r8d + addl %r14d,%r8d // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1447,8 +1447,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 20(%rsp),%r12d addl 48(%rsp),%r12d @@ -1460,43 +1460,43 @@ xorl %eax,%r13d rorl $9,%r14d - xorl %ecx,%r15d + xorl %ecx,%r15d // f^g movl %r12d,48(%rsp) xorl %r8d,%r14d - andl %eax,%r15d + andl %eax,%r15d // (f^g)&e rorl $5,%r13d - addl %edx,%r12d - xorl %ecx,%r15d + addl %edx,%r12d // T1+=h + xorl %ecx,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %eax,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %r8d,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %r8d,%r14d - xorl %r9d,%r15d - rorl $6,%r13d + xorl %r9d,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r9d,%edx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%edx - addl %r12d,%r11d - addl %r12d,%edx + xorl %edi,%edx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r11d // d+=T1 + addl %r12d,%edx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 56(%rsp),%r13d movl 44(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%edx + addl %r14d,%edx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1507,8 +1507,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 24(%rsp),%r12d addl 52(%rsp),%r12d @@ -1520,43 +1520,43 @@ xorl %r11d,%r13d rorl $9,%r14d - xorl %ebx,%edi + xorl %ebx,%edi // f^g movl %r12d,52(%rsp) xorl %edx,%r14d - andl %r11d,%edi + andl %r11d,%edi // (f^g)&e rorl $5,%r13d - addl %ecx,%r12d - xorl %ebx,%edi + addl %ecx,%r12d // T1+=h + xorl %ebx,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r11d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %edx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %edx,%r14d - xorl %r8d,%edi - rorl $6,%r13d + xorl %r8d,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %r8d,%ecx andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%ecx - addl %r12d,%r10d - addl %r12d,%ecx + xorl %r15d,%ecx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r10d // d+=T1 + addl %r12d,%ecx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 60(%rsp),%r13d movl 48(%rsp),%r15d movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ecx + addl %r14d,%ecx // modulo-scheduled h+=Sigma0(a) movl %r15d,%r14d rorl $2,%r15d @@ -1567,8 +1567,8 @@ shrl $10,%r14d rorl $17,%r15d - xorl %r13d,%r12d - xorl %r14d,%r15d + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%r15d // sigma1(X[(i+14)&0xf]) addl 28(%rsp),%r12d addl 56(%rsp),%r12d @@ -1580,43 +1580,43 @@ xorl %r10d,%r13d rorl $9,%r14d - xorl %eax,%r15d + xorl %eax,%r15d // f^g movl %r12d,56(%rsp) xorl %ecx,%r14d - andl %r10d,%r15d + andl %r10d,%r15d // (f^g)&e rorl $5,%r13d - addl %ebx,%r12d - xorl %eax,%r15d + addl %ebx,%r12d // T1+=h + xorl %eax,%r15d // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r10d,%r13d - addl %r15d,%r12d + addl %r15d,%r12d // T1+=Ch(e,f,g) movl %ecx,%r15d - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ecx,%r14d - xorl %edx,%r15d - rorl $6,%r13d + xorl %edx,%r15d // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %edx,%ebx andl %r15d,%edi - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %edi,%ebx - addl %r12d,%r9d - addl %r12d,%ebx + xorl %edi,%ebx // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r9d // d+=T1 + addl %r12d,%ebx // h+=T1 - leaq 4(%rbp),%rbp + leaq 4(%rbp),%rbp // round++ movl 0(%rsp),%r13d movl 52(%rsp),%edi movl %r13d,%r12d rorl $11,%r13d - addl %r14d,%ebx + addl %r14d,%ebx // modulo-scheduled h+=Sigma0(a) movl %edi,%r14d rorl $2,%edi @@ -1627,8 +1627,8 @@ shrl $10,%r14d rorl $17,%edi - xorl %r13d,%r12d - xorl %r14d,%edi + xorl %r13d,%r12d // sigma0(X[(i+1)&0xf]) + xorl %r14d,%edi // sigma1(X[(i+14)&0xf]) addl 32(%rsp),%r12d addl 60(%rsp),%r12d @@ -1640,42 +1640,42 @@ xorl %r9d,%r13d rorl $9,%r14d - xorl %r11d,%edi + xorl %r11d,%edi // f^g movl %r12d,60(%rsp) xorl %ebx,%r14d - andl %r9d,%edi + andl %r9d,%edi // (f^g)&e rorl $5,%r13d - addl %eax,%r12d - xorl %r11d,%edi + addl %eax,%r12d // T1+=h + xorl %r11d,%edi // Ch(e,f,g)=((f^g)&e)^g rorl $11,%r14d xorl %r9d,%r13d - addl %edi,%r12d + addl %edi,%r12d // T1+=Ch(e,f,g) movl %ebx,%edi - addl (%rbp),%r12d + addl (%rbp),%r12d // T1+=K[round] xorl %ebx,%r14d - xorl %ecx,%edi - rorl $6,%r13d + xorl %ecx,%edi // a^b, b^c in next round + rorl $6,%r13d // Sigma1(e) movl %ecx,%eax andl %edi,%r15d - rorl $2,%r14d - addl %r13d,%r12d + rorl $2,%r14d // Sigma0(a) + addl %r13d,%r12d // T1+=Sigma1(e) - xorl %r15d,%eax - addl %r12d,%r8d - addl %r12d,%eax + xorl %r15d,%eax // h=Maj(a,b,c)=Ch(a^b,c,b) + addl %r12d,%r8d // d+=T1 + addl %r12d,%eax // h+=T1 - leaq 20(%rbp),%rbp + leaq 20(%rbp),%rbp // round++ cmpb $0,3(%rbp) jnz .Lrounds_16_xx movq 64+0(%rsp),%rdi - addl %r14d,%eax + addl %r14d,%eax // modulo-scheduled h+=Sigma0(a) leaq 64(%rsi),%rsi addl 0(%rdi),%eax @@ -1772,16 +1772,16 @@ .cfi_startproc _CET_ENDBR leaq K256+128(%rip),%rcx - movdqu (%rdi),%xmm1 - movdqu 16(%rdi),%xmm2 - movdqa 512-128(%rcx),%xmm7 + movdqu (%rdi),%xmm1 // DCBA + movdqu 16(%rdi),%xmm2 // HGFE + movdqa 512-128(%rcx),%xmm7 // byte swap mask - pshufd $0x1b,%xmm1,%xmm0 - pshufd $0xb1,%xmm1,%xmm1 - pshufd $0x1b,%xmm2,%xmm2 - movdqa %xmm7,%xmm8 - palignr $8,%xmm2,%xmm1 - punpcklqdq %xmm0,%xmm2 + pshufd $0x1b,%xmm1,%xmm0 // ABCD + pshufd $0xb1,%xmm1,%xmm1 // CDAB + pshufd $0x1b,%xmm2,%xmm2 // EFGH + movdqa %xmm7,%xmm8 // offload + palignr $8,%xmm2,%xmm1 // ABEF + punpcklqdq %xmm0,%xmm2 // CDGH jmp .Loop_shaext .align 16 @@ -1795,17 +1795,17 @@ movdqa 0-128(%rcx),%xmm0 paddd %xmm3,%xmm0 pshufb %xmm7,%xmm4 - movdqa %xmm2,%xmm10 - sha256rnds2 %xmm1,%xmm2 + movdqa %xmm2,%xmm10 // offload + sha256rnds2 %xmm1,%xmm2 // 0-3 pshufd $0x0e,%xmm0,%xmm0 nop - movdqa %xmm1,%xmm9 + movdqa %xmm1,%xmm9 // offload sha256rnds2 %xmm2,%xmm1 movdqa 32-128(%rcx),%xmm0 paddd %xmm4,%xmm0 pshufb %xmm7,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 4-7 pshufd $0x0e,%xmm0,%xmm0 leaq 64(%rsi),%rsi sha256msg1 %xmm4,%xmm3 @@ -1814,7 +1814,7 @@ movdqa 64-128(%rcx),%xmm0 paddd %xmm5,%xmm0 pshufb %xmm7,%xmm6 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 8-11 pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm6,%xmm7 palignr $4,%xmm5,%xmm7 @@ -1826,7 +1826,7 @@ movdqa 96-128(%rcx),%xmm0 paddd %xmm6,%xmm0 sha256msg2 %xmm6,%xmm3 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 12-15 pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm3,%xmm7 palignr $4,%xmm6,%xmm7 @@ -1837,7 +1837,7 @@ movdqa 128-128(%rcx),%xmm0 paddd %xmm3,%xmm0 sha256msg2 %xmm3,%xmm4 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm4,%xmm7 palignr $4,%xmm3,%xmm7 @@ -1848,7 +1848,7 @@ movdqa 160-128(%rcx),%xmm0 paddd %xmm4,%xmm0 sha256msg2 %xmm4,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm5,%xmm7 palignr $4,%xmm4,%xmm7 @@ -1859,7 +1859,7 @@ movdqa 192-128(%rcx),%xmm0 paddd %xmm5,%xmm0 sha256msg2 %xmm5,%xmm6 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm6,%xmm7 palignr $4,%xmm5,%xmm7 @@ -1870,7 +1870,7 @@ movdqa 224-128(%rcx),%xmm0 paddd %xmm6,%xmm0 sha256msg2 %xmm6,%xmm3 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm3,%xmm7 palignr $4,%xmm6,%xmm7 @@ -1881,7 +1881,7 @@ movdqa 256-128(%rcx),%xmm0 paddd %xmm3,%xmm0 sha256msg2 %xmm3,%xmm4 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm4,%xmm7 palignr $4,%xmm3,%xmm7 @@ -1892,7 +1892,7 @@ movdqa 288-128(%rcx),%xmm0 paddd %xmm4,%xmm0 sha256msg2 %xmm4,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm5,%xmm7 palignr $4,%xmm4,%xmm7 @@ -1903,7 +1903,7 @@ movdqa 320-128(%rcx),%xmm0 paddd %xmm5,%xmm0 sha256msg2 %xmm5,%xmm6 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm6,%xmm7 palignr $4,%xmm5,%xmm7 @@ -1914,7 +1914,7 @@ movdqa 352-128(%rcx),%xmm0 paddd %xmm6,%xmm0 sha256msg2 %xmm6,%xmm3 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm3,%xmm7 palignr $4,%xmm6,%xmm7 @@ -1925,7 +1925,7 @@ movdqa 384-128(%rcx),%xmm0 paddd %xmm3,%xmm0 sha256msg2 %xmm3,%xmm4 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 16-19... pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm4,%xmm7 palignr $4,%xmm3,%xmm7 @@ -1936,7 +1936,7 @@ movdqa 416-128(%rcx),%xmm0 paddd %xmm4,%xmm0 sha256msg2 %xmm4,%xmm5 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 52-55 pshufd $0x0e,%xmm0,%xmm0 movdqa %xmm5,%xmm7 palignr $4,%xmm4,%xmm7 @@ -1945,7 +1945,7 @@ movdqa 448-128(%rcx),%xmm0 paddd %xmm5,%xmm0 - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 56-59 pshufd $0x0e,%xmm0,%xmm0 sha256msg2 %xmm5,%xmm6 movdqa %xmm8,%xmm7 @@ -1954,7 +1954,7 @@ movdqa 480-128(%rcx),%xmm0 paddd %xmm6,%xmm0 nop - sha256rnds2 %xmm1,%xmm2 + sha256rnds2 %xmm1,%xmm2 // 60-63 pshufd $0x0e,%xmm0,%xmm0 decq %rdx nop @@ -1964,11 +1964,11 @@ paddd %xmm9,%xmm1 jnz .Loop_shaext - pshufd $0xb1,%xmm2,%xmm2 - pshufd $0x1b,%xmm1,%xmm7 - pshufd $0xb1,%xmm1,%xmm1 - punpckhqdq %xmm2,%xmm1 - palignr $8,%xmm7,%xmm2 + pshufd $0xb1,%xmm2,%xmm2 // DCHG + pshufd $0x1b,%xmm1,%xmm7 // FEBA + pshufd $0xb1,%xmm1,%xmm1 // BAFE + punpckhqdq %xmm2,%xmm1 // DCBA + palignr $8,%xmm7,%xmm2 // HGFE movdqu %xmm1,(%rdi) movdqu %xmm2,16(%rdi) @@ -1982,7 +1982,7 @@ sha256_block_data_order_ssse3: .cfi_startproc _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp .cfi_def_cfa_register %rax pushq %rbx .cfi_offset %rbx,-16 @@ -1996,14 +1996,14 @@ .cfi_offset %r14,-48 pushq %r15 .cfi_offset %r15,-56 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $96,%rsp - leaq (%rsi,%rdx,4),%rdx - andq $-64,%rsp - movq %rdi,64+0(%rsp) - movq %rsi,64+8(%rsp) - movq %rdx,64+16(%rsp) - movq %rax,88(%rsp) + leaq (%rsi,%rdx,4),%rdx // inp+num*16*4 + andq $-64,%rsp // align stack frame + movq %rdi,64+0(%rsp) // save ctx, 1st arg + movq %rsi,64+8(%rsp) // save inp, 2nd arh + movq %rdx,64+16(%rsp) // save end pointer, "3rd" arg + movq %rax,88(%rsp) // save copy of %rsp .cfi_escape 0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08 .Lprologue_ssse3: @@ -2015,8 +2015,8 @@ movl 20(%rdi),%r9d movl 24(%rdi),%r10d movl 28(%rdi),%r11d - - +// movdqa K256+512+32(%rip),%xmm8 +// movdqa K256+512+64(%rip),%xmm9 jmp .Lloop_ssse3 .align 16 .Lloop_ssse3: @@ -2043,14 +2043,14 @@ movdqa %xmm5,16(%rsp) movl %ebx,%edi movdqa %xmm6,32(%rsp) - xorl %ecx,%edi + xorl %ecx,%edi // magic movdqa %xmm7,48(%rsp) movl %r8d,%r13d jmp .Lssse3_00_47 .align 16 .Lssse3_00_47: - subq $-128,%rbp + subq $-128,%rbp // size optimization rorl $14,%r13d movdqa %xmm1,%xmm4 movl %r14d,%eax @@ -3097,7 +3097,7 @@ sha256_block_data_order_avx: .cfi_startproc _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp .cfi_def_cfa_register %rax pushq %rbx .cfi_offset %rbx,-16 @@ -3111,14 +3111,14 @@ .cfi_offset %r14,-48 pushq %r15 .cfi_offset %r15,-56 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $96,%rsp - leaq (%rsi,%rdx,4),%rdx - andq $-64,%rsp - movq %rdi,64+0(%rsp) - movq %rsi,64+8(%rsp) - movq %rdx,64+16(%rsp) - movq %rax,88(%rsp) + leaq (%rsi,%rdx,4),%rdx // inp+num*16*4 + andq $-64,%rsp // align stack frame + movq %rdi,64+0(%rsp) // save ctx, 1st arg + movq %rsi,64+8(%rsp) // save inp, 2nd arh + movq %rdx,64+16(%rsp) // save end pointer, "3rd" arg + movq %rax,88(%rsp) // save copy of %rsp .cfi_escape 0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08 .Lprologue_avx: @@ -3155,14 +3155,14 @@ vmovdqa %xmm5,16(%rsp) movl %ebx,%edi vmovdqa %xmm6,32(%rsp) - xorl %ecx,%edi + xorl %ecx,%edi // magic vmovdqa %xmm7,48(%rsp) movl %r8d,%r13d jmp .Lavx_00_47 .align 16 .Lavx_00_47: - subq $-128,%rbp + subq $-128,%rbp // size optimization vpalignr $4,%xmm0,%xmm1,%xmm4 shrdl $14,%r13d,%r13d movl %r14d,%eax
diff --git a/gen/bcm/sha256-x86_64-win.asm b/gen/bcm/sha256-x86_64-win.asm index 61b11d8..339f23a 100644 --- a/gen/bcm/sha256-x86_64-win.asm +++ b/gen/bcm/sha256-x86_64-win.asm
@@ -29,7 +29,7 @@ _CET_ENDBR - mov rax,rsp + mov rax,rsp ; copy %rsp push rbx @@ -43,14 +43,14 @@ push r15 - shl rdx,4 + shl rdx,4 ; num*16 sub rsp,16*4+4*8 - lea rdx,[rdx*4+rsi] - and rsp,-64 - mov QWORD[((64+0))+rsp],rdi - mov QWORD[((64+8))+rsp],rsi - mov QWORD[((64+16))+rsp],rdx - mov QWORD[88+rsp],rax + lea rdx,[rdx*4+rsi] ; inp+num*16*4 + and rsp,-64 ; align stack frame + mov QWORD[((64+0))+rsp],rdi ; save ctx, 1st arg + mov QWORD[((64+8))+rsp],rsi ; save inp, 2nd arh + mov QWORD[((64+16))+rsp],rdx ; save end pointer, "3rd" arg + mov QWORD[88+rsp],rax ; save copy of %rsp $L$prologue: @@ -68,7 +68,7 @@ $L$loop: mov edi,ebx lea rbp,[K256] - xor edi,ecx + xor edi,ecx ; magic mov r12d,DWORD[rsi] mov r13d,r8d mov r14d,eax @@ -78,38 +78,38 @@ xor r13d,r8d ror r14d,9 - xor r15d,r10d + xor r15d,r10d ; f^g mov DWORD[rsp],r12d xor r14d,eax - and r15d,r8d + and r15d,r8d ; (f^g)&e ror r13d,5 - add r12d,r11d - xor r15d,r10d + add r12d,r11d ; T1+=h + xor r15d,r10d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r8d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,eax - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,eax - xor r15d,ebx - ror r13d,6 + xor r15d,ebx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r11d,ebx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r11d,edi - add edx,r12d - add r11d,r12d + xor r11d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add edx,r12d ; d+=T1 + add r11d,r12d ; h+=T1 - lea rbp,[4+rbp] - add r11d,r14d + lea rbp,[4+rbp] ; round++ + add r11d,r14d ; h+=Sigma0(a) mov r12d,DWORD[4+rsi] mov r13d,edx mov r14d,r11d @@ -119,38 +119,38 @@ xor r13d,edx ror r14d,9 - xor edi,r9d + xor edi,r9d ; f^g mov DWORD[4+rsp],r12d xor r14d,r11d - and edi,edx + and edi,edx ; (f^g)&e ror r13d,5 - add r12d,r10d - xor edi,r9d + add r12d,r10d ; T1+=h + xor edi,r9d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,edx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r11d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r11d - xor edi,eax - ror r13d,6 + xor edi,eax ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r10d,eax and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r10d,r15d - add ecx,r12d - add r10d,r12d + xor r10d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ecx,r12d ; d+=T1 + add r10d,r12d ; h+=T1 - lea rbp,[4+rbp] - add r10d,r14d + lea rbp,[4+rbp] ; round++ + add r10d,r14d ; h+=Sigma0(a) mov r12d,DWORD[8+rsi] mov r13d,ecx mov r14d,r10d @@ -160,38 +160,38 @@ xor r13d,ecx ror r14d,9 - xor r15d,r8d + xor r15d,r8d ; f^g mov DWORD[8+rsp],r12d xor r14d,r10d - and r15d,ecx + and r15d,ecx ; (f^g)&e ror r13d,5 - add r12d,r9d - xor r15d,r8d + add r12d,r9d ; T1+=h + xor r15d,r8d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ecx - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r10d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r10d - xor r15d,r11d - ror r13d,6 + xor r15d,r11d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r9d,r11d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r9d,edi - add ebx,r12d - add r9d,r12d + xor r9d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ebx,r12d ; d+=T1 + add r9d,r12d ; h+=T1 - lea rbp,[4+rbp] - add r9d,r14d + lea rbp,[4+rbp] ; round++ + add r9d,r14d ; h+=Sigma0(a) mov r12d,DWORD[12+rsi] mov r13d,ebx mov r14d,r9d @@ -201,38 +201,38 @@ xor r13d,ebx ror r14d,9 - xor edi,edx + xor edi,edx ; f^g mov DWORD[12+rsp],r12d xor r14d,r9d - and edi,ebx + and edi,ebx ; (f^g)&e ror r13d,5 - add r12d,r8d - xor edi,edx + add r12d,r8d ; T1+=h + xor edi,edx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ebx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r9d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r9d - xor edi,r10d - ror r13d,6 + xor edi,r10d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r8d,r10d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r8d,r15d - add eax,r12d - add r8d,r12d + xor r8d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add eax,r12d ; d+=T1 + add r8d,r12d ; h+=T1 - lea rbp,[20+rbp] - add r8d,r14d + lea rbp,[20+rbp] ; round++ + add r8d,r14d ; h+=Sigma0(a) mov r12d,DWORD[16+rsi] mov r13d,eax mov r14d,r8d @@ -242,38 +242,38 @@ xor r13d,eax ror r14d,9 - xor r15d,ecx + xor r15d,ecx ; f^g mov DWORD[16+rsp],r12d xor r14d,r8d - and r15d,eax + and r15d,eax ; (f^g)&e ror r13d,5 - add r12d,edx - xor r15d,ecx + add r12d,edx ; T1+=h + xor r15d,ecx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,eax - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r8d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r8d - xor r15d,r9d - ror r13d,6 + xor r15d,r9d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov edx,r9d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor edx,edi - add r11d,r12d - add edx,r12d + xor edx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11d,r12d ; d+=T1 + add edx,r12d ; h+=T1 - lea rbp,[4+rbp] - add edx,r14d + lea rbp,[4+rbp] ; round++ + add edx,r14d ; h+=Sigma0(a) mov r12d,DWORD[20+rsi] mov r13d,r11d mov r14d,edx @@ -283,38 +283,38 @@ xor r13d,r11d ror r14d,9 - xor edi,ebx + xor edi,ebx ; f^g mov DWORD[20+rsp],r12d xor r14d,edx - and edi,r11d + and edi,r11d ; (f^g)&e ror r13d,5 - add r12d,ecx - xor edi,ebx + add r12d,ecx ; T1+=h + xor edi,ebx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r11d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,edx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,edx - xor edi,r8d - ror r13d,6 + xor edi,r8d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ecx,r8d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ecx,r15d - add r10d,r12d - add ecx,r12d + xor ecx,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10d,r12d ; d+=T1 + add ecx,r12d ; h+=T1 - lea rbp,[4+rbp] - add ecx,r14d + lea rbp,[4+rbp] ; round++ + add ecx,r14d ; h+=Sigma0(a) mov r12d,DWORD[24+rsi] mov r13d,r10d mov r14d,ecx @@ -324,38 +324,38 @@ xor r13d,r10d ror r14d,9 - xor r15d,eax + xor r15d,eax ; f^g mov DWORD[24+rsp],r12d xor r14d,ecx - and r15d,r10d + and r15d,r10d ; (f^g)&e ror r13d,5 - add r12d,ebx - xor r15d,eax + add r12d,ebx ; T1+=h + xor r15d,eax ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r10d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,ecx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ecx - xor r15d,edx - ror r13d,6 + xor r15d,edx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ebx,edx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ebx,edi - add r9d,r12d - add ebx,r12d + xor ebx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9d,r12d ; d+=T1 + add ebx,r12d ; h+=T1 - lea rbp,[4+rbp] - add ebx,r14d + lea rbp,[4+rbp] ; round++ + add ebx,r14d ; h+=Sigma0(a) mov r12d,DWORD[28+rsi] mov r13d,r9d mov r14d,ebx @@ -365,38 +365,38 @@ xor r13d,r9d ror r14d,9 - xor edi,r11d + xor edi,r11d ; f^g mov DWORD[28+rsp],r12d xor r14d,ebx - and edi,r9d + and edi,r9d ; (f^g)&e ror r13d,5 - add r12d,eax - xor edi,r11d + add r12d,eax ; T1+=h + xor edi,r11d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r9d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,ebx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ebx - xor edi,ecx - ror r13d,6 + xor edi,ecx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov eax,ecx and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor eax,r15d - add r8d,r12d - add eax,r12d + xor eax,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8d,r12d ; d+=T1 + add eax,r12d ; h+=T1 - lea rbp,[20+rbp] - add eax,r14d + lea rbp,[20+rbp] ; round++ + add eax,r14d ; h+=Sigma0(a) mov r12d,DWORD[32+rsi] mov r13d,r8d mov r14d,eax @@ -406,38 +406,38 @@ xor r13d,r8d ror r14d,9 - xor r15d,r10d + xor r15d,r10d ; f^g mov DWORD[32+rsp],r12d xor r14d,eax - and r15d,r8d + and r15d,r8d ; (f^g)&e ror r13d,5 - add r12d,r11d - xor r15d,r10d + add r12d,r11d ; T1+=h + xor r15d,r10d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r8d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,eax - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,eax - xor r15d,ebx - ror r13d,6 + xor r15d,ebx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r11d,ebx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r11d,edi - add edx,r12d - add r11d,r12d + xor r11d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add edx,r12d ; d+=T1 + add r11d,r12d ; h+=T1 - lea rbp,[4+rbp] - add r11d,r14d + lea rbp,[4+rbp] ; round++ + add r11d,r14d ; h+=Sigma0(a) mov r12d,DWORD[36+rsi] mov r13d,edx mov r14d,r11d @@ -447,38 +447,38 @@ xor r13d,edx ror r14d,9 - xor edi,r9d + xor edi,r9d ; f^g mov DWORD[36+rsp],r12d xor r14d,r11d - and edi,edx + and edi,edx ; (f^g)&e ror r13d,5 - add r12d,r10d - xor edi,r9d + add r12d,r10d ; T1+=h + xor edi,r9d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,edx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r11d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r11d - xor edi,eax - ror r13d,6 + xor edi,eax ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r10d,eax and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r10d,r15d - add ecx,r12d - add r10d,r12d + xor r10d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ecx,r12d ; d+=T1 + add r10d,r12d ; h+=T1 - lea rbp,[4+rbp] - add r10d,r14d + lea rbp,[4+rbp] ; round++ + add r10d,r14d ; h+=Sigma0(a) mov r12d,DWORD[40+rsi] mov r13d,ecx mov r14d,r10d @@ -488,38 +488,38 @@ xor r13d,ecx ror r14d,9 - xor r15d,r8d + xor r15d,r8d ; f^g mov DWORD[40+rsp],r12d xor r14d,r10d - and r15d,ecx + and r15d,ecx ; (f^g)&e ror r13d,5 - add r12d,r9d - xor r15d,r8d + add r12d,r9d ; T1+=h + xor r15d,r8d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ecx - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r10d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r10d - xor r15d,r11d - ror r13d,6 + xor r15d,r11d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r9d,r11d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r9d,edi - add ebx,r12d - add r9d,r12d + xor r9d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ebx,r12d ; d+=T1 + add r9d,r12d ; h+=T1 - lea rbp,[4+rbp] - add r9d,r14d + lea rbp,[4+rbp] ; round++ + add r9d,r14d ; h+=Sigma0(a) mov r12d,DWORD[44+rsi] mov r13d,ebx mov r14d,r9d @@ -529,38 +529,38 @@ xor r13d,ebx ror r14d,9 - xor edi,edx + xor edi,edx ; f^g mov DWORD[44+rsp],r12d xor r14d,r9d - and edi,ebx + and edi,ebx ; (f^g)&e ror r13d,5 - add r12d,r8d - xor edi,edx + add r12d,r8d ; T1+=h + xor edi,edx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ebx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r9d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r9d - xor edi,r10d - ror r13d,6 + xor edi,r10d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r8d,r10d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r8d,r15d - add eax,r12d - add r8d,r12d + xor r8d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add eax,r12d ; d+=T1 + add r8d,r12d ; h+=T1 - lea rbp,[20+rbp] - add r8d,r14d + lea rbp,[20+rbp] ; round++ + add r8d,r14d ; h+=Sigma0(a) mov r12d,DWORD[48+rsi] mov r13d,eax mov r14d,r8d @@ -570,38 +570,38 @@ xor r13d,eax ror r14d,9 - xor r15d,ecx + xor r15d,ecx ; f^g mov DWORD[48+rsp],r12d xor r14d,r8d - and r15d,eax + and r15d,eax ; (f^g)&e ror r13d,5 - add r12d,edx - xor r15d,ecx + add r12d,edx ; T1+=h + xor r15d,ecx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,eax - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r8d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r8d - xor r15d,r9d - ror r13d,6 + xor r15d,r9d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov edx,r9d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor edx,edi - add r11d,r12d - add edx,r12d + xor edx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11d,r12d ; d+=T1 + add edx,r12d ; h+=T1 - lea rbp,[4+rbp] - add edx,r14d + lea rbp,[4+rbp] ; round++ + add edx,r14d ; h+=Sigma0(a) mov r12d,DWORD[52+rsi] mov r13d,r11d mov r14d,edx @@ -611,38 +611,38 @@ xor r13d,r11d ror r14d,9 - xor edi,ebx + xor edi,ebx ; f^g mov DWORD[52+rsp],r12d xor r14d,edx - and edi,r11d + and edi,r11d ; (f^g)&e ror r13d,5 - add r12d,ecx - xor edi,ebx + add r12d,ecx ; T1+=h + xor edi,ebx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r11d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,edx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,edx - xor edi,r8d - ror r13d,6 + xor edi,r8d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ecx,r8d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ecx,r15d - add r10d,r12d - add ecx,r12d + xor ecx,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10d,r12d ; d+=T1 + add ecx,r12d ; h+=T1 - lea rbp,[4+rbp] - add ecx,r14d + lea rbp,[4+rbp] ; round++ + add ecx,r14d ; h+=Sigma0(a) mov r12d,DWORD[56+rsi] mov r13d,r10d mov r14d,ecx @@ -652,38 +652,38 @@ xor r13d,r10d ror r14d,9 - xor r15d,eax + xor r15d,eax ; f^g mov DWORD[56+rsp],r12d xor r14d,ecx - and r15d,r10d + and r15d,r10d ; (f^g)&e ror r13d,5 - add r12d,ebx - xor r15d,eax + add r12d,ebx ; T1+=h + xor r15d,eax ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r10d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,ecx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ecx - xor r15d,edx - ror r13d,6 + xor r15d,edx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ebx,edx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ebx,edi - add r9d,r12d - add ebx,r12d + xor ebx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9d,r12d ; d+=T1 + add ebx,r12d ; h+=T1 - lea rbp,[4+rbp] - add ebx,r14d + lea rbp,[4+rbp] ; round++ + add ebx,r14d ; h+=Sigma0(a) mov r12d,DWORD[60+rsi] mov r13d,r9d mov r14d,ebx @@ -693,37 +693,37 @@ xor r13d,r9d ror r14d,9 - xor edi,r11d + xor edi,r11d ; f^g mov DWORD[60+rsp],r12d xor r14d,ebx - and edi,r9d + and edi,r9d ; (f^g)&e ror r13d,5 - add r12d,eax - xor edi,r11d + add r12d,eax ; T1+=h + xor edi,r11d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r9d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,ebx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ebx - xor edi,ecx - ror r13d,6 + xor edi,ecx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov eax,ecx and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor eax,r15d - add r8d,r12d - add eax,r12d + xor eax,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8d,r12d ; d+=T1 + add eax,r12d ; h+=T1 - lea rbp,[20+rbp] + lea rbp,[20+rbp] ; round++ jmp NEAR $L$rounds_16_xx ALIGN 16 $L$rounds_16_xx: @@ -732,7 +732,7 @@ mov r12d,r13d ror r13d,11 - add eax,r14d + add eax,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -743,8 +743,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[36+rsp] add r12d,DWORD[rsp] @@ -756,43 +756,43 @@ xor r13d,r8d ror r14d,9 - xor r15d,r10d + xor r15d,r10d ; f^g mov DWORD[rsp],r12d xor r14d,eax - and r15d,r8d + and r15d,r8d ; (f^g)&e ror r13d,5 - add r12d,r11d - xor r15d,r10d + add r12d,r11d ; T1+=h + xor r15d,r10d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r8d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,eax - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,eax - xor r15d,ebx - ror r13d,6 + xor r15d,ebx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r11d,ebx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r11d,edi - add edx,r12d - add r11d,r12d + xor r11d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add edx,r12d ; d+=T1 + add r11d,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[8+rsp] mov edi,DWORD[60+rsp] mov r12d,r13d ror r13d,11 - add r11d,r14d + add r11d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -803,8 +803,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[40+rsp] add r12d,DWORD[4+rsp] @@ -816,43 +816,43 @@ xor r13d,edx ror r14d,9 - xor edi,r9d + xor edi,r9d ; f^g mov DWORD[4+rsp],r12d xor r14d,r11d - and edi,edx + and edi,edx ; (f^g)&e ror r13d,5 - add r12d,r10d - xor edi,r9d + add r12d,r10d ; T1+=h + xor edi,r9d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,edx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r11d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r11d - xor edi,eax - ror r13d,6 + xor edi,eax ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r10d,eax and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r10d,r15d - add ecx,r12d - add r10d,r12d + xor r10d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ecx,r12d ; d+=T1 + add r10d,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[12+rsp] mov r15d,DWORD[rsp] mov r12d,r13d ror r13d,11 - add r10d,r14d + add r10d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -863,8 +863,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[44+rsp] add r12d,DWORD[8+rsp] @@ -876,43 +876,43 @@ xor r13d,ecx ror r14d,9 - xor r15d,r8d + xor r15d,r8d ; f^g mov DWORD[8+rsp],r12d xor r14d,r10d - and r15d,ecx + and r15d,ecx ; (f^g)&e ror r13d,5 - add r12d,r9d - xor r15d,r8d + add r12d,r9d ; T1+=h + xor r15d,r8d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ecx - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r10d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r10d - xor r15d,r11d - ror r13d,6 + xor r15d,r11d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r9d,r11d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r9d,edi - add ebx,r12d - add r9d,r12d + xor r9d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ebx,r12d ; d+=T1 + add r9d,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[16+rsp] mov edi,DWORD[4+rsp] mov r12d,r13d ror r13d,11 - add r9d,r14d + add r9d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -923,8 +923,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[48+rsp] add r12d,DWORD[12+rsp] @@ -936,43 +936,43 @@ xor r13d,ebx ror r14d,9 - xor edi,edx + xor edi,edx ; f^g mov DWORD[12+rsp],r12d xor r14d,r9d - and edi,ebx + and edi,ebx ; (f^g)&e ror r13d,5 - add r12d,r8d - xor edi,edx + add r12d,r8d ; T1+=h + xor edi,edx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ebx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r9d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r9d - xor edi,r10d - ror r13d,6 + xor edi,r10d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r8d,r10d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r8d,r15d - add eax,r12d - add r8d,r12d + xor r8d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add eax,r12d ; d+=T1 + add r8d,r12d ; h+=T1 - lea rbp,[20+rbp] + lea rbp,[20+rbp] ; round++ mov r13d,DWORD[20+rsp] mov r15d,DWORD[8+rsp] mov r12d,r13d ror r13d,11 - add r8d,r14d + add r8d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -983,8 +983,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[52+rsp] add r12d,DWORD[16+rsp] @@ -996,43 +996,43 @@ xor r13d,eax ror r14d,9 - xor r15d,ecx + xor r15d,ecx ; f^g mov DWORD[16+rsp],r12d xor r14d,r8d - and r15d,eax + and r15d,eax ; (f^g)&e ror r13d,5 - add r12d,edx - xor r15d,ecx + add r12d,edx ; T1+=h + xor r15d,ecx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,eax - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r8d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r8d - xor r15d,r9d - ror r13d,6 + xor r15d,r9d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov edx,r9d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor edx,edi - add r11d,r12d - add edx,r12d + xor edx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11d,r12d ; d+=T1 + add edx,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[24+rsp] mov edi,DWORD[12+rsp] mov r12d,r13d ror r13d,11 - add edx,r14d + add edx,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -1043,8 +1043,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[56+rsp] add r12d,DWORD[20+rsp] @@ -1056,43 +1056,43 @@ xor r13d,r11d ror r14d,9 - xor edi,ebx + xor edi,ebx ; f^g mov DWORD[20+rsp],r12d xor r14d,edx - and edi,r11d + and edi,r11d ; (f^g)&e ror r13d,5 - add r12d,ecx - xor edi,ebx + add r12d,ecx ; T1+=h + xor edi,ebx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r11d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,edx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,edx - xor edi,r8d - ror r13d,6 + xor edi,r8d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ecx,r8d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ecx,r15d - add r10d,r12d - add ecx,r12d + xor ecx,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10d,r12d ; d+=T1 + add ecx,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[28+rsp] mov r15d,DWORD[16+rsp] mov r12d,r13d ror r13d,11 - add ecx,r14d + add ecx,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -1103,8 +1103,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[60+rsp] add r12d,DWORD[24+rsp] @@ -1116,43 +1116,43 @@ xor r13d,r10d ror r14d,9 - xor r15d,eax + xor r15d,eax ; f^g mov DWORD[24+rsp],r12d xor r14d,ecx - and r15d,r10d + and r15d,r10d ; (f^g)&e ror r13d,5 - add r12d,ebx - xor r15d,eax + add r12d,ebx ; T1+=h + xor r15d,eax ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r10d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,ecx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ecx - xor r15d,edx - ror r13d,6 + xor r15d,edx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ebx,edx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ebx,edi - add r9d,r12d - add ebx,r12d + xor ebx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9d,r12d ; d+=T1 + add ebx,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[32+rsp] mov edi,DWORD[20+rsp] mov r12d,r13d ror r13d,11 - add ebx,r14d + add ebx,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -1163,8 +1163,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[rsp] add r12d,DWORD[28+rsp] @@ -1176,43 +1176,43 @@ xor r13d,r9d ror r14d,9 - xor edi,r11d + xor edi,r11d ; f^g mov DWORD[28+rsp],r12d xor r14d,ebx - and edi,r9d + and edi,r9d ; (f^g)&e ror r13d,5 - add r12d,eax - xor edi,r11d + add r12d,eax ; T1+=h + xor edi,r11d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r9d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,ebx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ebx - xor edi,ecx - ror r13d,6 + xor edi,ecx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov eax,ecx and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor eax,r15d - add r8d,r12d - add eax,r12d + xor eax,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8d,r12d ; d+=T1 + add eax,r12d ; h+=T1 - lea rbp,[20+rbp] + lea rbp,[20+rbp] ; round++ mov r13d,DWORD[36+rsp] mov r15d,DWORD[24+rsp] mov r12d,r13d ror r13d,11 - add eax,r14d + add eax,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -1223,8 +1223,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[4+rsp] add r12d,DWORD[32+rsp] @@ -1236,43 +1236,43 @@ xor r13d,r8d ror r14d,9 - xor r15d,r10d + xor r15d,r10d ; f^g mov DWORD[32+rsp],r12d xor r14d,eax - and r15d,r8d + and r15d,r8d ; (f^g)&e ror r13d,5 - add r12d,r11d - xor r15d,r10d + add r12d,r11d ; T1+=h + xor r15d,r10d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r8d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,eax - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,eax - xor r15d,ebx - ror r13d,6 + xor r15d,ebx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r11d,ebx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r11d,edi - add edx,r12d - add r11d,r12d + xor r11d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add edx,r12d ; d+=T1 + add r11d,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[40+rsp] mov edi,DWORD[28+rsp] mov r12d,r13d ror r13d,11 - add r11d,r14d + add r11d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -1283,8 +1283,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[8+rsp] add r12d,DWORD[36+rsp] @@ -1296,43 +1296,43 @@ xor r13d,edx ror r14d,9 - xor edi,r9d + xor edi,r9d ; f^g mov DWORD[36+rsp],r12d xor r14d,r11d - and edi,edx + and edi,edx ; (f^g)&e ror r13d,5 - add r12d,r10d - xor edi,r9d + add r12d,r10d ; T1+=h + xor edi,r9d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,edx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r11d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r11d - xor edi,eax - ror r13d,6 + xor edi,eax ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r10d,eax and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r10d,r15d - add ecx,r12d - add r10d,r12d + xor r10d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ecx,r12d ; d+=T1 + add r10d,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[44+rsp] mov r15d,DWORD[32+rsp] mov r12d,r13d ror r13d,11 - add r10d,r14d + add r10d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -1343,8 +1343,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[12+rsp] add r12d,DWORD[40+rsp] @@ -1356,43 +1356,43 @@ xor r13d,ecx ror r14d,9 - xor r15d,r8d + xor r15d,r8d ; f^g mov DWORD[40+rsp],r12d xor r14d,r10d - and r15d,ecx + and r15d,ecx ; (f^g)&e ror r13d,5 - add r12d,r9d - xor r15d,r8d + add r12d,r9d ; T1+=h + xor r15d,r8d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ecx - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r10d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r10d - xor r15d,r11d - ror r13d,6 + xor r15d,r11d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r9d,r11d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r9d,edi - add ebx,r12d - add r9d,r12d + xor r9d,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add ebx,r12d ; d+=T1 + add r9d,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[48+rsp] mov edi,DWORD[36+rsp] mov r12d,r13d ror r13d,11 - add r9d,r14d + add r9d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -1403,8 +1403,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[16+rsp] add r12d,DWORD[44+rsp] @@ -1416,43 +1416,43 @@ xor r13d,ebx ror r14d,9 - xor edi,edx + xor edi,edx ; f^g mov DWORD[44+rsp],r12d xor r14d,r9d - and edi,ebx + and edi,ebx ; (f^g)&e ror r13d,5 - add r12d,r8d - xor edi,edx + add r12d,r8d ; T1+=h + xor edi,edx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,ebx - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,r9d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r9d - xor edi,r10d - ror r13d,6 + xor edi,r10d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov r8d,r10d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor r8d,r15d - add eax,r12d - add r8d,r12d + xor r8d,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add eax,r12d ; d+=T1 + add r8d,r12d ; h+=T1 - lea rbp,[20+rbp] + lea rbp,[20+rbp] ; round++ mov r13d,DWORD[52+rsp] mov r15d,DWORD[40+rsp] mov r12d,r13d ror r13d,11 - add r8d,r14d + add r8d,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -1463,8 +1463,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[20+rsp] add r12d,DWORD[48+rsp] @@ -1476,43 +1476,43 @@ xor r13d,eax ror r14d,9 - xor r15d,ecx + xor r15d,ecx ; f^g mov DWORD[48+rsp],r12d xor r14d,r8d - and r15d,eax + and r15d,eax ; (f^g)&e ror r13d,5 - add r12d,edx - xor r15d,ecx + add r12d,edx ; T1+=h + xor r15d,ecx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,eax - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,r8d - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,r8d - xor r15d,r9d - ror r13d,6 + xor r15d,r9d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov edx,r9d and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor edx,edi - add r11d,r12d - add edx,r12d + xor edx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11d,r12d ; d+=T1 + add edx,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[56+rsp] mov edi,DWORD[44+rsp] mov r12d,r13d ror r13d,11 - add edx,r14d + add edx,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -1523,8 +1523,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[24+rsp] add r12d,DWORD[52+rsp] @@ -1536,43 +1536,43 @@ xor r13d,r11d ror r14d,9 - xor edi,ebx + xor edi,ebx ; f^g mov DWORD[52+rsp],r12d xor r14d,edx - and edi,r11d + and edi,r11d ; (f^g)&e ror r13d,5 - add r12d,ecx - xor edi,ebx + add r12d,ecx ; T1+=h + xor edi,ebx ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r11d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,edx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,edx - xor edi,r8d - ror r13d,6 + xor edi,r8d ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ecx,r8d and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ecx,r15d - add r10d,r12d - add ecx,r12d + xor ecx,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10d,r12d ; d+=T1 + add ecx,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[60+rsp] mov r15d,DWORD[48+rsp] mov r12d,r13d ror r13d,11 - add ecx,r14d + add ecx,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,r15d ror r15d,2 @@ -1583,8 +1583,8 @@ shr r14d,10 ror r15d,17 - xor r12d,r13d - xor r15d,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor r15d,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[28+rsp] add r12d,DWORD[56+rsp] @@ -1596,43 +1596,43 @@ xor r13d,r10d ror r14d,9 - xor r15d,eax + xor r15d,eax ; f^g mov DWORD[56+rsp],r12d xor r14d,ecx - and r15d,r10d + and r15d,r10d ; (f^g)&e ror r13d,5 - add r12d,ebx - xor r15d,eax + add r12d,ebx ; T1+=h + xor r15d,eax ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r10d - add r12d,r15d + add r12d,r15d ; T1+=Ch(e,f,g) mov r15d,ecx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ecx - xor r15d,edx - ror r13d,6 + xor r15d,edx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov ebx,edx and edi,r15d - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor ebx,edi - add r9d,r12d - add ebx,r12d + xor ebx,edi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9d,r12d ; d+=T1 + add ebx,r12d ; h+=T1 - lea rbp,[4+rbp] + lea rbp,[4+rbp] ; round++ mov r13d,DWORD[rsp] mov edi,DWORD[52+rsp] mov r12d,r13d ror r13d,11 - add ebx,r14d + add ebx,r14d ; modulo-scheduled h+=Sigma0(a) mov r14d,edi ror edi,2 @@ -1643,8 +1643,8 @@ shr r14d,10 ror edi,17 - xor r12d,r13d - xor edi,r14d + xor r12d,r13d ; sigma0(X[(i+1)&0xf]) + xor edi,r14d ; sigma1(X[(i+14)&0xf]) add r12d,DWORD[32+rsp] add r12d,DWORD[60+rsp] @@ -1656,42 +1656,42 @@ xor r13d,r9d ror r14d,9 - xor edi,r11d + xor edi,r11d ; f^g mov DWORD[60+rsp],r12d xor r14d,ebx - and edi,r9d + and edi,r9d ; (f^g)&e ror r13d,5 - add r12d,eax - xor edi,r11d + add r12d,eax ; T1+=h + xor edi,r11d ; Ch(e,f,g)=((f^g)&e)^g ror r14d,11 xor r13d,r9d - add r12d,edi + add r12d,edi ; T1+=Ch(e,f,g) mov edi,ebx - add r12d,DWORD[rbp] + add r12d,DWORD[rbp] ; T1+=K[round] xor r14d,ebx - xor edi,ecx - ror r13d,6 + xor edi,ecx ; a^b, b^c in next round + ror r13d,6 ; Sigma1(e) mov eax,ecx and r15d,edi - ror r14d,2 - add r12d,r13d + ror r14d,2 ; Sigma0(a) + add r12d,r13d ; T1+=Sigma1(e) - xor eax,r15d - add r8d,r12d - add eax,r12d + xor eax,r15d ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8d,r12d ; d+=T1 + add eax,r12d ; h+=T1 - lea rbp,[20+rbp] + lea rbp,[20+rbp] ; round++ cmp BYTE[3+rbp],0 jnz NEAR $L$rounds_16_xx mov rdi,QWORD[((64+0))+rsp] - add eax,r14d + add eax,r14d ; modulo-scheduled h+=Sigma0(a) lea rsi,[64+rsi] add eax,DWORD[rdi] @@ -1810,16 +1810,16 @@ movaps XMMWORD[(-8-16)+rax],xmm10 $L$prologue_shaext: lea rcx,[((K256+128))] - movdqu xmm1,XMMWORD[rdi] - movdqu xmm2,XMMWORD[16+rdi] - movdqa xmm7,XMMWORD[((512-128))+rcx] + movdqu xmm1,XMMWORD[rdi] ; DCBA + movdqu xmm2,XMMWORD[16+rdi] ; HGFE + movdqa xmm7,XMMWORD[((512-128))+rcx] ; byte swap mask - pshufd xmm0,xmm1,0x1b - pshufd xmm1,xmm1,0xb1 - pshufd xmm2,xmm2,0x1b - movdqa xmm8,xmm7 - palignr xmm1,xmm2,8 - punpcklqdq xmm2,xmm0 + pshufd xmm0,xmm1,0x1b ; ABCD + pshufd xmm1,xmm1,0xb1 ; CDAB + pshufd xmm2,xmm2,0x1b ; EFGH + movdqa xmm8,xmm7 ; offload + palignr xmm1,xmm2,8 ; ABEF + punpcklqdq xmm2,xmm0 ; CDGH jmp NEAR $L$oop_shaext ALIGN 16 @@ -1833,17 +1833,17 @@ movdqa xmm0,XMMWORD[((0-128))+rcx] paddd xmm0,xmm3 pshufb xmm4,xmm7 - movdqa xmm10,xmm2 - sha256rnds2 xmm2,xmm1 + movdqa xmm10,xmm2 ; offload + sha256rnds2 xmm2,xmm1 ; 0-3 pshufd xmm0,xmm0,0x0e nop - movdqa xmm9,xmm1 + movdqa xmm9,xmm1 ; offload sha256rnds2 xmm1,xmm2 movdqa xmm0,XMMWORD[((32-128))+rcx] paddd xmm0,xmm4 pshufb xmm5,xmm7 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 4-7 pshufd xmm0,xmm0,0x0e lea rsi,[64+rsi] sha256msg1 xmm3,xmm4 @@ -1852,7 +1852,7 @@ movdqa xmm0,XMMWORD[((64-128))+rcx] paddd xmm0,xmm5 pshufb xmm6,xmm7 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 8-11 pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm6 palignr xmm7,xmm5,4 @@ -1864,7 +1864,7 @@ movdqa xmm0,XMMWORD[((96-128))+rcx] paddd xmm0,xmm6 sha256msg2 xmm3,xmm6 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 12-15 pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm3 palignr xmm7,xmm6,4 @@ -1875,7 +1875,7 @@ movdqa xmm0,XMMWORD[((128-128))+rcx] paddd xmm0,xmm3 sha256msg2 xmm4,xmm3 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm4 palignr xmm7,xmm3,4 @@ -1886,7 +1886,7 @@ movdqa xmm0,XMMWORD[((160-128))+rcx] paddd xmm0,xmm4 sha256msg2 xmm5,xmm4 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm5 palignr xmm7,xmm4,4 @@ -1897,7 +1897,7 @@ movdqa xmm0,XMMWORD[((192-128))+rcx] paddd xmm0,xmm5 sha256msg2 xmm6,xmm5 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm6 palignr xmm7,xmm5,4 @@ -1908,7 +1908,7 @@ movdqa xmm0,XMMWORD[((224-128))+rcx] paddd xmm0,xmm6 sha256msg2 xmm3,xmm6 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm3 palignr xmm7,xmm6,4 @@ -1919,7 +1919,7 @@ movdqa xmm0,XMMWORD[((256-128))+rcx] paddd xmm0,xmm3 sha256msg2 xmm4,xmm3 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm4 palignr xmm7,xmm3,4 @@ -1930,7 +1930,7 @@ movdqa xmm0,XMMWORD[((288-128))+rcx] paddd xmm0,xmm4 sha256msg2 xmm5,xmm4 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm5 palignr xmm7,xmm4,4 @@ -1941,7 +1941,7 @@ movdqa xmm0,XMMWORD[((320-128))+rcx] paddd xmm0,xmm5 sha256msg2 xmm6,xmm5 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm6 palignr xmm7,xmm5,4 @@ -1952,7 +1952,7 @@ movdqa xmm0,XMMWORD[((352-128))+rcx] paddd xmm0,xmm6 sha256msg2 xmm3,xmm6 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm3 palignr xmm7,xmm6,4 @@ -1963,7 +1963,7 @@ movdqa xmm0,XMMWORD[((384-128))+rcx] paddd xmm0,xmm3 sha256msg2 xmm4,xmm3 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 16-19... pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm4 palignr xmm7,xmm3,4 @@ -1974,7 +1974,7 @@ movdqa xmm0,XMMWORD[((416-128))+rcx] paddd xmm0,xmm4 sha256msg2 xmm5,xmm4 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 52-55 pshufd xmm0,xmm0,0x0e movdqa xmm7,xmm5 palignr xmm7,xmm4,4 @@ -1983,7 +1983,7 @@ movdqa xmm0,XMMWORD[((448-128))+rcx] paddd xmm0,xmm5 - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 56-59 pshufd xmm0,xmm0,0x0e sha256msg2 xmm6,xmm5 movdqa xmm7,xmm8 @@ -1992,7 +1992,7 @@ movdqa xmm0,XMMWORD[((480-128))+rcx] paddd xmm0,xmm6 nop - sha256rnds2 xmm2,xmm1 + sha256rnds2 xmm2,xmm1 ; 60-63 pshufd xmm0,xmm0,0x0e dec rdx nop @@ -2002,11 +2002,11 @@ paddd xmm1,xmm9 jnz NEAR $L$oop_shaext - pshufd xmm2,xmm2,0xb1 - pshufd xmm7,xmm1,0x1b - pshufd xmm1,xmm1,0xb1 - punpckhqdq xmm1,xmm2 - palignr xmm2,xmm7,8 + pshufd xmm2,xmm2,0xb1 ; DCHG + pshufd xmm7,xmm1,0x1b ; FEBA + pshufd xmm1,xmm1,0xb1 ; BAFE + punpckhqdq xmm1,xmm2 ; DCBA + palignr xmm2,xmm7,8 ; HGFE movdqu XMMWORD[rdi],xmm1 movdqu XMMWORD[16+rdi],xmm2 @@ -2037,7 +2037,7 @@ _CET_ENDBR - mov rax,rsp + mov rax,rsp ; copy %rsp push rbx @@ -2051,14 +2051,14 @@ push r15 - shl rdx,4 + shl rdx,4 ; num*16 sub rsp,160 - lea rdx,[rdx*4+rsi] - and rsp,-64 - mov QWORD[((64+0))+rsp],rdi - mov QWORD[((64+8))+rsp],rsi - mov QWORD[((64+16))+rsp],rdx - mov QWORD[88+rsp],rax + lea rdx,[rdx*4+rsi] ; inp+num*16*4 + and rsp,-64 ; align stack frame + mov QWORD[((64+0))+rsp],rdi ; save ctx, 1st arg + mov QWORD[((64+8))+rsp],rsi ; save inp, 2nd arh + mov QWORD[((64+16))+rsp],rdx ; save end pointer, "3rd" arg + mov QWORD[88+rsp],rax ; save copy of %rsp movaps XMMWORD[(64+32)+rsp],xmm6 movaps XMMWORD[(64+48)+rsp],xmm7 @@ -2074,8 +2074,8 @@ mov r9d,DWORD[20+rdi] mov r10d,DWORD[24+rdi] mov r11d,DWORD[28+rdi] - - +; movdqa K256+512+32(%rip),%xmm8 +; movdqa K256+512+64(%rip),%xmm9 jmp NEAR $L$loop_ssse3 ALIGN 16 $L$loop_ssse3: @@ -2102,14 +2102,14 @@ movdqa XMMWORD[16+rsp],xmm5 mov edi,ebx movdqa XMMWORD[32+rsp],xmm6 - xor edi,ecx + xor edi,ecx ; magic movdqa XMMWORD[48+rsp],xmm7 mov r13d,r8d jmp NEAR $L$ssse3_00_47 ALIGN 16 $L$ssse3_00_47: - sub rbp,-128 + sub rbp,-128 ; size optimization ror r13d,14 movdqa xmm4,xmm1 mov eax,r14d @@ -3170,7 +3170,7 @@ _CET_ENDBR - mov rax,rsp + mov rax,rsp ; copy %rsp push rbx @@ -3184,14 +3184,14 @@ push r15 - shl rdx,4 + shl rdx,4 ; num*16 sub rsp,160 - lea rdx,[rdx*4+rsi] - and rsp,-64 - mov QWORD[((64+0))+rsp],rdi - mov QWORD[((64+8))+rsp],rsi - mov QWORD[((64+16))+rsp],rdx - mov QWORD[88+rsp],rax + lea rdx,[rdx*4+rsi] ; inp+num*16*4 + and rsp,-64 ; align stack frame + mov QWORD[((64+0))+rsp],rdi ; save ctx, 1st arg + mov QWORD[((64+8))+rsp],rsi ; save inp, 2nd arh + mov QWORD[((64+16))+rsp],rdx ; save end pointer, "3rd" arg + mov QWORD[88+rsp],rax ; save copy of %rsp movaps XMMWORD[(64+32)+rsp],xmm6 movaps XMMWORD[(64+48)+rsp],xmm7 @@ -3232,14 +3232,14 @@ vmovdqa XMMWORD[16+rsp],xmm5 mov edi,ebx vmovdqa XMMWORD[32+rsp],xmm6 - xor edi,ecx + xor edi,ecx ; magic vmovdqa XMMWORD[48+rsp],xmm7 mov r13d,r8d jmp NEAR $L$avx_00_47 ALIGN 16 $L$avx_00_47: - sub rbp,-128 + sub rbp,-128 ; size optimization vpalignr xmm4,xmm1,xmm0,4 shrd r13d,r13d,14 mov eax,r14d @@ -4265,25 +4265,25 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HanderlData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$in_prologue - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$in_prologue - mov rsi,rax - mov rax,QWORD[((64+24))+rax] + mov rsi,rax ; put aside Rsp + mov rax,QWORD[((64+24))+rax] ; pull 88(%rsp) mov rbx,QWORD[((-8))+rax] mov rbp,QWORD[((-16))+rax] @@ -4291,49 +4291,49 @@ mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R15 lea r10,[$L$epilogue] cmp rbx,r10 - jb NEAR $L$in_prologue + jb NEAR $L$in_prologue ; non-AVX code - lea rsi,[((64+32))+rsi] - lea rdi,[512+r8] + lea rsi,[((64+32))+rsi] ; Xmm6- save area + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,8 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq $L$in_prologue: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -4360,21 +4360,21 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip lea r10,[$L$prologue_shaext] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<.Lprologue jb NEAR $L$in_prologue lea r10,[$L$epilogue_shaext] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip>=.Lepilogue jae NEAR $L$in_prologue lea rsi,[((-8-80))+rax] - lea rdi,[512+r8] + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,10 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq jmp NEAR $L$in_prologue @@ -4397,18 +4397,18 @@ $L$SEH_info_sha256_block_data_order_nohw: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$prologue wrt ..imagebase,$L$epilogue wrt ..imagebase + DD $L$prologue wrt ..imagebase,$L$epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_sha256_block_data_order_hw: DB 9,0,0,0 DD shaext_handler wrt ..imagebase $L$SEH_info_sha256_block_data_order_ssse3: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase + DD $L$prologue_ssse3 wrt ..imagebase,$L$epilogue_ssse3 wrt ..imagebase ; HandlerData[] $L$SEH_info_sha256_block_data_order_avx: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase + DD $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase ; HandlerData[] %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/bcm/sha512-x86_64-apple.S b/gen/bcm/sha512-x86_64-apple.S index 9d9d3d0..5524b91 100644 --- a/gen/bcm/sha512-x86_64-apple.S +++ b/gen/bcm/sha512-x86_64-apple.S
@@ -13,7 +13,7 @@ _sha512_block_data_order_nohw: _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp pushq %rbx @@ -27,14 +27,14 @@ pushq %r15 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $128+32,%rsp - leaq (%rsi,%rdx,8),%rdx - andq $-64,%rsp - movq %rdi,128+0(%rsp) - movq %rsi,128+8(%rsp) - movq %rdx,128+16(%rsp) - movq %rax,152(%rsp) + leaq (%rsi,%rdx,8),%rdx // inp+num*16*8 + andq $-64,%rsp // align stack frame + movq %rdi,128+0(%rsp) // save ctx, 1st arg + movq %rsi,128+8(%rsp) // save inp, 2nd arh + movq %rdx,128+16(%rsp) // save end pointer, "3rd" arg + movq %rax,152(%rsp) // save copy of %rsp L$prologue: @@ -52,7 +52,7 @@ L$loop: movq %rbx,%rdi leaq K512(%rip),%rbp - xorq %rcx,%rdi + xorq %rcx,%rdi // magic movq 0(%rsi),%r12 movq %r8,%r13 movq %rax,%r14 @@ -62,38 +62,38 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,0(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r11 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r11 // h+=Sigma0(a) movq 8(%rsi),%r12 movq %rdx,%r13 movq %r11,%r14 @@ -103,38 +103,38 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,8(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r10 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r10 // h+=Sigma0(a) movq 16(%rsi),%r12 movq %rcx,%r13 movq %r10,%r14 @@ -144,38 +144,38 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,16(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r9 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r9 // h+=Sigma0(a) movq 24(%rsi),%r12 movq %rbx,%r13 movq %r9,%r14 @@ -185,38 +185,38 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,24(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r8 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r8 // h+=Sigma0(a) movq 32(%rsi),%r12 movq %rax,%r13 movq %r8,%r14 @@ -226,38 +226,38 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,32(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rdx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rdx // h+=Sigma0(a) movq 40(%rsi),%r12 movq %r11,%r13 movq %rdx,%r14 @@ -267,38 +267,38 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,40(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%rcx + leaq 24(%rbp),%rbp // round++ + addq %r14,%rcx // h+=Sigma0(a) movq 48(%rsi),%r12 movq %r10,%r13 movq %rcx,%r14 @@ -308,38 +308,38 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,48(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rbx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rbx // h+=Sigma0(a) movq 56(%rsi),%r12 movq %r9,%r13 movq %rbx,%r14 @@ -349,38 +349,38 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,56(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%rax + leaq 24(%rbp),%rbp // round++ + addq %r14,%rax // h+=Sigma0(a) movq 64(%rsi),%r12 movq %r8,%r13 movq %rax,%r14 @@ -390,38 +390,38 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,64(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r11 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r11 // h+=Sigma0(a) movq 72(%rsi),%r12 movq %rdx,%r13 movq %r11,%r14 @@ -431,38 +431,38 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,72(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r10 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r10 // h+=Sigma0(a) movq 80(%rsi),%r12 movq %rcx,%r13 movq %r10,%r14 @@ -472,38 +472,38 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,80(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r9 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r9 // h+=Sigma0(a) movq 88(%rsi),%r12 movq %rbx,%r13 movq %r9,%r14 @@ -513,38 +513,38 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,88(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r8 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r8 // h+=Sigma0(a) movq 96(%rsi),%r12 movq %rax,%r13 movq %r8,%r14 @@ -554,38 +554,38 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,96(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rdx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rdx // h+=Sigma0(a) movq 104(%rsi),%r12 movq %r11,%r13 movq %rdx,%r14 @@ -595,38 +595,38 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,104(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%rcx + leaq 24(%rbp),%rbp // round++ + addq %r14,%rcx // h+=Sigma0(a) movq 112(%rsi),%r12 movq %r10,%r13 movq %rcx,%r14 @@ -636,38 +636,38 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,112(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rbx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rbx // h+=Sigma0(a) movq 120(%rsi),%r12 movq %r9,%r13 movq %rbx,%r14 @@ -677,37 +677,37 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,120(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ jmp L$rounds_16_xx .p2align 4 L$rounds_16_xx: @@ -716,7 +716,7 @@ movq %r13,%r12 rorq $7,%r13 - addq %r14,%rax + addq %r14,%rax // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -727,8 +727,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 72(%rsp),%r12 addq 0(%rsp),%r12 @@ -740,43 +740,43 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,0(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 16(%rsp),%r13 movq 120(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r11 + addq %r14,%r11 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -787,8 +787,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 80(%rsp),%r12 addq 8(%rsp),%r12 @@ -800,43 +800,43 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,8(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 24(%rsp),%r13 movq 0(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r10 + addq %r14,%r10 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -847,8 +847,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 88(%rsp),%r12 addq 16(%rsp),%r12 @@ -860,43 +860,43 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,16(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 32(%rsp),%r13 movq 8(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r9 + addq %r14,%r9 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -907,8 +907,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 96(%rsp),%r12 addq 24(%rsp),%r12 @@ -920,43 +920,43 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,24(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 40(%rsp),%r13 movq 16(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r8 + addq %r14,%r8 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -967,8 +967,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 104(%rsp),%r12 addq 32(%rsp),%r12 @@ -980,43 +980,43 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,32(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 48(%rsp),%r13 movq 24(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rdx + addq %r14,%rdx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1027,8 +1027,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 112(%rsp),%r12 addq 40(%rsp),%r12 @@ -1040,43 +1040,43 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,40(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 56(%rsp),%r13 movq 32(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%rcx + addq %r14,%rcx // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1087,8 +1087,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 120(%rsp),%r12 addq 48(%rsp),%r12 @@ -1100,43 +1100,43 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,48(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 64(%rsp),%r13 movq 40(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rbx + addq %r14,%rbx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1147,8 +1147,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 0(%rsp),%r12 addq 56(%rsp),%r12 @@ -1160,43 +1160,43 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,56(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 72(%rsp),%r13 movq 48(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%rax + addq %r14,%rax // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1207,8 +1207,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 8(%rsp),%r12 addq 64(%rsp),%r12 @@ -1220,43 +1220,43 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,64(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 80(%rsp),%r13 movq 56(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r11 + addq %r14,%r11 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1267,8 +1267,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 16(%rsp),%r12 addq 72(%rsp),%r12 @@ -1280,43 +1280,43 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,72(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 88(%rsp),%r13 movq 64(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r10 + addq %r14,%r10 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1327,8 +1327,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 24(%rsp),%r12 addq 80(%rsp),%r12 @@ -1340,43 +1340,43 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,80(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 96(%rsp),%r13 movq 72(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r9 + addq %r14,%r9 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1387,8 +1387,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 32(%rsp),%r12 addq 88(%rsp),%r12 @@ -1400,43 +1400,43 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,88(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 104(%rsp),%r13 movq 80(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r8 + addq %r14,%r8 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1447,8 +1447,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 40(%rsp),%r12 addq 96(%rsp),%r12 @@ -1460,43 +1460,43 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,96(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 112(%rsp),%r13 movq 88(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rdx + addq %r14,%rdx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1507,8 +1507,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 48(%rsp),%r12 addq 104(%rsp),%r12 @@ -1520,43 +1520,43 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,104(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 120(%rsp),%r13 movq 96(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%rcx + addq %r14,%rcx // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1567,8 +1567,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 56(%rsp),%r12 addq 112(%rsp),%r12 @@ -1580,43 +1580,43 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,112(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 0(%rsp),%r13 movq 104(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rbx + addq %r14,%rbx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1627,8 +1627,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 64(%rsp),%r12 addq 120(%rsp),%r12 @@ -1640,42 +1640,42 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,120(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ cmpb $0,7(%rbp) jnz L$rounds_16_xx movq 128+0(%rsp),%rdi - addq %r14,%rax + addq %r14,%rax // modulo-scheduled h+=Sigma0(a) leaq 128(%rsi),%rsi addq 0(%rdi),%rax @@ -1816,7 +1816,7 @@ _sha512_block_data_order_avx: _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp pushq %rbx @@ -1830,14 +1830,14 @@ pushq %r15 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $160,%rsp - leaq (%rsi,%rdx,8),%rdx - andq $-64,%rsp - movq %rdi,128+0(%rsp) - movq %rsi,128+8(%rsp) - movq %rdx,128+16(%rsp) - movq %rax,152(%rsp) + leaq (%rsi,%rdx,8),%rdx // inp+num*16*8 + andq $-64,%rsp // align stack frame + movq %rdi,128+0(%rsp) // save ctx, 1st arg + movq %rsi,128+8(%rsp) // save inp, 2nd arh + movq %rdx,128+16(%rsp) // save end pointer, "3rd" arg + movq %rax,152(%rsp) // save copy of %rsp L$prologue_avx: @@ -1855,7 +1855,7 @@ L$loop_avx: vmovdqa K512+1280(%rip),%xmm11 vmovdqu 0(%rsi),%xmm0 - leaq K512+128(%rip),%rbp + leaq K512+128(%rip),%rbp // size optimization vmovdqu 16(%rsi),%xmm1 vmovdqu 32(%rsi),%xmm2 vpshufb %xmm11,%xmm0,%xmm0 @@ -1888,7 +1888,7 @@ vmovdqa %xmm9,80(%rsp) movq %rbx,%rdi vmovdqa %xmm10,96(%rsp) - xorq %rcx,%rdi + xorq %rcx,%rdi // magic vmovdqa %xmm11,112(%rsp) movq %r8,%r13 jmp L$avx_00_47
diff --git a/gen/bcm/sha512-x86_64-linux.S b/gen/bcm/sha512-x86_64-linux.S index bbef943..f701e88 100644 --- a/gen/bcm/sha512-x86_64-linux.S +++ b/gen/bcm/sha512-x86_64-linux.S
@@ -13,7 +13,7 @@ sha512_block_data_order_nohw: .cfi_startproc _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp .cfi_def_cfa_register %rax pushq %rbx .cfi_offset %rbx,-16 @@ -27,14 +27,14 @@ .cfi_offset %r14,-48 pushq %r15 .cfi_offset %r15,-56 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $128+32,%rsp - leaq (%rsi,%rdx,8),%rdx - andq $-64,%rsp - movq %rdi,128+0(%rsp) - movq %rsi,128+8(%rsp) - movq %rdx,128+16(%rsp) - movq %rax,152(%rsp) + leaq (%rsi,%rdx,8),%rdx // inp+num*16*8 + andq $-64,%rsp // align stack frame + movq %rdi,128+0(%rsp) // save ctx, 1st arg + movq %rsi,128+8(%rsp) // save inp, 2nd arh + movq %rdx,128+16(%rsp) // save end pointer, "3rd" arg + movq %rax,152(%rsp) // save copy of %rsp .cfi_escape 0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08 .Lprologue: @@ -52,7 +52,7 @@ .Lloop: movq %rbx,%rdi leaq K512(%rip),%rbp - xorq %rcx,%rdi + xorq %rcx,%rdi // magic movq 0(%rsi),%r12 movq %r8,%r13 movq %rax,%r14 @@ -62,38 +62,38 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,0(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r11 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r11 // h+=Sigma0(a) movq 8(%rsi),%r12 movq %rdx,%r13 movq %r11,%r14 @@ -103,38 +103,38 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,8(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r10 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r10 // h+=Sigma0(a) movq 16(%rsi),%r12 movq %rcx,%r13 movq %r10,%r14 @@ -144,38 +144,38 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,16(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r9 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r9 // h+=Sigma0(a) movq 24(%rsi),%r12 movq %rbx,%r13 movq %r9,%r14 @@ -185,38 +185,38 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,24(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r8 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r8 // h+=Sigma0(a) movq 32(%rsi),%r12 movq %rax,%r13 movq %r8,%r14 @@ -226,38 +226,38 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,32(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rdx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rdx // h+=Sigma0(a) movq 40(%rsi),%r12 movq %r11,%r13 movq %rdx,%r14 @@ -267,38 +267,38 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,40(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%rcx + leaq 24(%rbp),%rbp // round++ + addq %r14,%rcx // h+=Sigma0(a) movq 48(%rsi),%r12 movq %r10,%r13 movq %rcx,%r14 @@ -308,38 +308,38 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,48(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rbx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rbx // h+=Sigma0(a) movq 56(%rsi),%r12 movq %r9,%r13 movq %rbx,%r14 @@ -349,38 +349,38 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,56(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%rax + leaq 24(%rbp),%rbp // round++ + addq %r14,%rax // h+=Sigma0(a) movq 64(%rsi),%r12 movq %r8,%r13 movq %rax,%r14 @@ -390,38 +390,38 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,64(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r11 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r11 // h+=Sigma0(a) movq 72(%rsi),%r12 movq %rdx,%r13 movq %r11,%r14 @@ -431,38 +431,38 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,72(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r10 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r10 // h+=Sigma0(a) movq 80(%rsi),%r12 movq %rcx,%r13 movq %r10,%r14 @@ -472,38 +472,38 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,80(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%r9 + leaq 8(%rbp),%rbp // round++ + addq %r14,%r9 // h+=Sigma0(a) movq 88(%rsi),%r12 movq %rbx,%r13 movq %r9,%r14 @@ -513,38 +513,38 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,88(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%r8 + leaq 24(%rbp),%rbp // round++ + addq %r14,%r8 // h+=Sigma0(a) movq 96(%rsi),%r12 movq %rax,%r13 movq %r8,%r14 @@ -554,38 +554,38 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,96(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rdx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rdx // h+=Sigma0(a) movq 104(%rsi),%r12 movq %r11,%r13 movq %rdx,%r14 @@ -595,38 +595,38 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,104(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp - addq %r14,%rcx + leaq 24(%rbp),%rbp // round++ + addq %r14,%rcx // h+=Sigma0(a) movq 112(%rsi),%r12 movq %r10,%r13 movq %rcx,%r14 @@ -636,38 +636,38 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,112(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp - addq %r14,%rbx + leaq 8(%rbp),%rbp // round++ + addq %r14,%rbx // h+=Sigma0(a) movq 120(%rsi),%r12 movq %r9,%r13 movq %rbx,%r14 @@ -677,37 +677,37 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,120(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ jmp .Lrounds_16_xx .align 16 .Lrounds_16_xx: @@ -716,7 +716,7 @@ movq %r13,%r12 rorq $7,%r13 - addq %r14,%rax + addq %r14,%rax // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -727,8 +727,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 72(%rsp),%r12 addq 0(%rsp),%r12 @@ -740,43 +740,43 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,0(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 16(%rsp),%r13 movq 120(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r11 + addq %r14,%r11 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -787,8 +787,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 80(%rsp),%r12 addq 8(%rsp),%r12 @@ -800,43 +800,43 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,8(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 24(%rsp),%r13 movq 0(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r10 + addq %r14,%r10 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -847,8 +847,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 88(%rsp),%r12 addq 16(%rsp),%r12 @@ -860,43 +860,43 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,16(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 32(%rsp),%r13 movq 8(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r9 + addq %r14,%r9 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -907,8 +907,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 96(%rsp),%r12 addq 24(%rsp),%r12 @@ -920,43 +920,43 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,24(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 40(%rsp),%r13 movq 16(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r8 + addq %r14,%r8 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -967,8 +967,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 104(%rsp),%r12 addq 32(%rsp),%r12 @@ -980,43 +980,43 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,32(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 48(%rsp),%r13 movq 24(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rdx + addq %r14,%rdx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1027,8 +1027,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 112(%rsp),%r12 addq 40(%rsp),%r12 @@ -1040,43 +1040,43 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,40(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 56(%rsp),%r13 movq 32(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%rcx + addq %r14,%rcx // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1087,8 +1087,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 120(%rsp),%r12 addq 48(%rsp),%r12 @@ -1100,43 +1100,43 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,48(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 64(%rsp),%r13 movq 40(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rbx + addq %r14,%rbx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1147,8 +1147,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 0(%rsp),%r12 addq 56(%rsp),%r12 @@ -1160,43 +1160,43 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,56(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 72(%rsp),%r13 movq 48(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%rax + addq %r14,%rax // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1207,8 +1207,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 8(%rsp),%r12 addq 64(%rsp),%r12 @@ -1220,43 +1220,43 @@ xorq %r8,%r13 rorq $5,%r14 - xorq %r10,%r15 + xorq %r10,%r15 // f^g movq %r12,64(%rsp) xorq %rax,%r14 - andq %r8,%r15 + andq %r8,%r15 // (f^g)&e rorq $4,%r13 - addq %r11,%r12 - xorq %r10,%r15 + addq %r11,%r12 // T1+=h + xorq %r10,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r8,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rax,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rax,%r14 - xorq %rbx,%r15 - rorq $14,%r13 + xorq %rbx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rbx,%r11 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r11 - addq %r12,%rdx - addq %r12,%r11 + xorq %rdi,%r11 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rdx // d+=T1 + addq %r12,%r11 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 80(%rsp),%r13 movq 56(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r11 + addq %r14,%r11 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1267,8 +1267,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 16(%rsp),%r12 addq 72(%rsp),%r12 @@ -1280,43 +1280,43 @@ xorq %rdx,%r13 rorq $5,%r14 - xorq %r9,%rdi + xorq %r9,%rdi // f^g movq %r12,72(%rsp) xorq %r11,%r14 - andq %rdx,%rdi + andq %rdx,%rdi // (f^g)&e rorq $4,%r13 - addq %r10,%r12 - xorq %r9,%rdi + addq %r10,%r12 // T1+=h + xorq %r9,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rdx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r11,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r11,%r14 - xorq %rax,%rdi - rorq $14,%r13 + xorq %rax,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rax,%r10 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r10 - addq %r12,%rcx - addq %r12,%r10 + xorq %r15,%r10 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rcx // d+=T1 + addq %r12,%r10 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 88(%rsp),%r13 movq 64(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r10 + addq %r14,%r10 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1327,8 +1327,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 24(%rsp),%r12 addq 80(%rsp),%r12 @@ -1340,43 +1340,43 @@ xorq %rcx,%r13 rorq $5,%r14 - xorq %r8,%r15 + xorq %r8,%r15 // f^g movq %r12,80(%rsp) xorq %r10,%r14 - andq %rcx,%r15 + andq %rcx,%r15 // (f^g)&e rorq $4,%r13 - addq %r9,%r12 - xorq %r8,%r15 + addq %r9,%r12 // T1+=h + xorq %r8,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rcx,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r10,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r10,%r14 - xorq %r11,%r15 - rorq $14,%r13 + xorq %r11,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r11,%r9 andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%r9 - addq %r12,%rbx - addq %r12,%r9 + xorq %rdi,%r9 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rbx // d+=T1 + addq %r12,%r9 // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 96(%rsp),%r13 movq 72(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%r9 + addq %r14,%r9 // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1387,8 +1387,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 32(%rsp),%r12 addq 88(%rsp),%r12 @@ -1400,43 +1400,43 @@ xorq %rbx,%r13 rorq $5,%r14 - xorq %rdx,%rdi + xorq %rdx,%rdi // f^g movq %r12,88(%rsp) xorq %r9,%r14 - andq %rbx,%rdi + andq %rbx,%rdi // (f^g)&e rorq $4,%r13 - addq %r8,%r12 - xorq %rdx,%rdi + addq %r8,%r12 // T1+=h + xorq %rdx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rbx,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %r9,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r9,%r14 - xorq %r10,%rdi - rorq $14,%r13 + xorq %r10,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r10,%r8 andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%r8 - addq %r12,%rax - addq %r12,%r8 + xorq %r15,%r8 // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%rax // d+=T1 + addq %r12,%r8 // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 104(%rsp),%r13 movq 80(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%r8 + addq %r14,%r8 // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1447,8 +1447,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 40(%rsp),%r12 addq 96(%rsp),%r12 @@ -1460,43 +1460,43 @@ xorq %rax,%r13 rorq $5,%r14 - xorq %rcx,%r15 + xorq %rcx,%r15 // f^g movq %r12,96(%rsp) xorq %r8,%r14 - andq %rax,%r15 + andq %rax,%r15 // (f^g)&e rorq $4,%r13 - addq %rdx,%r12 - xorq %rcx,%r15 + addq %rdx,%r12 // T1+=h + xorq %rcx,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %rax,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %r8,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %r8,%r14 - xorq %r9,%r15 - rorq $14,%r13 + xorq %r9,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r9,%rdx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rdx - addq %r12,%r11 - addq %r12,%rdx + xorq %rdi,%rdx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r11 // d+=T1 + addq %r12,%rdx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 112(%rsp),%r13 movq 88(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rdx + addq %r14,%rdx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1507,8 +1507,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 48(%rsp),%r12 addq 104(%rsp),%r12 @@ -1520,43 +1520,43 @@ xorq %r11,%r13 rorq $5,%r14 - xorq %rbx,%rdi + xorq %rbx,%rdi // f^g movq %r12,104(%rsp) xorq %rdx,%r14 - andq %r11,%rdi + andq %r11,%rdi // (f^g)&e rorq $4,%r13 - addq %rcx,%r12 - xorq %rbx,%rdi + addq %rcx,%r12 // T1+=h + xorq %rbx,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r11,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rdx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rdx,%r14 - xorq %r8,%rdi - rorq $14,%r13 + xorq %r8,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %r8,%rcx andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rcx - addq %r12,%r10 - addq %r12,%rcx + xorq %r15,%rcx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r10 // d+=T1 + addq %r12,%rcx // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ movq 120(%rsp),%r13 movq 96(%rsp),%r15 movq %r13,%r12 rorq $7,%r13 - addq %r14,%rcx + addq %r14,%rcx // modulo-scheduled h+=Sigma0(a) movq %r15,%r14 rorq $42,%r15 @@ -1567,8 +1567,8 @@ shrq $6,%r14 rorq $19,%r15 - xorq %r13,%r12 - xorq %r14,%r15 + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%r15 // sigma1(X[(i+14)&0xf]) addq 56(%rsp),%r12 addq 112(%rsp),%r12 @@ -1580,43 +1580,43 @@ xorq %r10,%r13 rorq $5,%r14 - xorq %rax,%r15 + xorq %rax,%r15 // f^g movq %r12,112(%rsp) xorq %rcx,%r14 - andq %r10,%r15 + andq %r10,%r15 // (f^g)&e rorq $4,%r13 - addq %rbx,%r12 - xorq %rax,%r15 + addq %rbx,%r12 // T1+=h + xorq %rax,%r15 // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r10,%r13 - addq %r15,%r12 + addq %r15,%r12 // T1+=Ch(e,f,g) movq %rcx,%r15 - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rcx,%r14 - xorq %rdx,%r15 - rorq $14,%r13 + xorq %rdx,%r15 // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rdx,%rbx andq %r15,%rdi - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %rdi,%rbx - addq %r12,%r9 - addq %r12,%rbx + xorq %rdi,%rbx // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r9 // d+=T1 + addq %r12,%rbx // h+=T1 - leaq 8(%rbp),%rbp + leaq 8(%rbp),%rbp // round++ movq 0(%rsp),%r13 movq 104(%rsp),%rdi movq %r13,%r12 rorq $7,%r13 - addq %r14,%rbx + addq %r14,%rbx // modulo-scheduled h+=Sigma0(a) movq %rdi,%r14 rorq $42,%rdi @@ -1627,8 +1627,8 @@ shrq $6,%r14 rorq $19,%rdi - xorq %r13,%r12 - xorq %r14,%rdi + xorq %r13,%r12 // sigma0(X[(i+1)&0xf]) + xorq %r14,%rdi // sigma1(X[(i+14)&0xf]) addq 64(%rsp),%r12 addq 120(%rsp),%r12 @@ -1640,42 +1640,42 @@ xorq %r9,%r13 rorq $5,%r14 - xorq %r11,%rdi + xorq %r11,%rdi // f^g movq %r12,120(%rsp) xorq %rbx,%r14 - andq %r9,%rdi + andq %r9,%rdi // (f^g)&e rorq $4,%r13 - addq %rax,%r12 - xorq %r11,%rdi + addq %rax,%r12 // T1+=h + xorq %r11,%rdi // Ch(e,f,g)=((f^g)&e)^g rorq $6,%r14 xorq %r9,%r13 - addq %rdi,%r12 + addq %rdi,%r12 // T1+=Ch(e,f,g) movq %rbx,%rdi - addq (%rbp),%r12 + addq (%rbp),%r12 // T1+=K[round] xorq %rbx,%r14 - xorq %rcx,%rdi - rorq $14,%r13 + xorq %rcx,%rdi // a^b, b^c in next round + rorq $14,%r13 // Sigma1(e) movq %rcx,%rax andq %rdi,%r15 - rorq $28,%r14 - addq %r13,%r12 + rorq $28,%r14 // Sigma0(a) + addq %r13,%r12 // T1+=Sigma1(e) - xorq %r15,%rax - addq %r12,%r8 - addq %r12,%rax + xorq %r15,%rax // h=Maj(a,b,c)=Ch(a^b,c,b) + addq %r12,%r8 // d+=T1 + addq %r12,%rax // h+=T1 - leaq 24(%rbp),%rbp + leaq 24(%rbp),%rbp // round++ cmpb $0,7(%rbp) jnz .Lrounds_16_xx movq 128+0(%rsp),%rdi - addq %r14,%rax + addq %r14,%rax // modulo-scheduled h+=Sigma0(a) leaq 128(%rsi),%rsi addq 0(%rdi),%rax @@ -1815,7 +1815,7 @@ sha512_block_data_order_avx: .cfi_startproc _CET_ENDBR - movq %rsp,%rax + movq %rsp,%rax // copy %rsp .cfi_def_cfa_register %rax pushq %rbx .cfi_offset %rbx,-16 @@ -1829,14 +1829,14 @@ .cfi_offset %r14,-48 pushq %r15 .cfi_offset %r15,-56 - shlq $4,%rdx + shlq $4,%rdx // num*16 subq $160,%rsp - leaq (%rsi,%rdx,8),%rdx - andq $-64,%rsp - movq %rdi,128+0(%rsp) - movq %rsi,128+8(%rsp) - movq %rdx,128+16(%rsp) - movq %rax,152(%rsp) + leaq (%rsi,%rdx,8),%rdx // inp+num*16*8 + andq $-64,%rsp // align stack frame + movq %rdi,128+0(%rsp) // save ctx, 1st arg + movq %rsi,128+8(%rsp) // save inp, 2nd arh + movq %rdx,128+16(%rsp) // save end pointer, "3rd" arg + movq %rax,152(%rsp) // save copy of %rsp .cfi_escape 0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08 .Lprologue_avx: @@ -1854,7 +1854,7 @@ .Lloop_avx: vmovdqa K512+1280(%rip),%xmm11 vmovdqu 0(%rsi),%xmm0 - leaq K512+128(%rip),%rbp + leaq K512+128(%rip),%rbp // size optimization vmovdqu 16(%rsi),%xmm1 vmovdqu 32(%rsi),%xmm2 vpshufb %xmm11,%xmm0,%xmm0 @@ -1887,7 +1887,7 @@ vmovdqa %xmm9,80(%rsp) movq %rbx,%rdi vmovdqa %xmm10,96(%rsp) - xorq %rcx,%rdi + xorq %rcx,%rdi // magic vmovdqa %xmm11,112(%rsp) movq %r8,%r13 jmp .Lavx_00_47
diff --git a/gen/bcm/sha512-x86_64-win.asm b/gen/bcm/sha512-x86_64-win.asm index 53cabe6..2e6760c 100644 --- a/gen/bcm/sha512-x86_64-win.asm +++ b/gen/bcm/sha512-x86_64-win.asm
@@ -29,7 +29,7 @@ _CET_ENDBR - mov rax,rsp + mov rax,rsp ; copy %rsp push rbx @@ -43,14 +43,14 @@ push r15 - shl rdx,4 + shl rdx,4 ; num*16 sub rsp,16*8+4*8 - lea rdx,[rdx*8+rsi] - and rsp,-64 - mov QWORD[((128+0))+rsp],rdi - mov QWORD[((128+8))+rsp],rsi - mov QWORD[((128+16))+rsp],rdx - mov QWORD[152+rsp],rax + lea rdx,[rdx*8+rsi] ; inp+num*16*8 + and rsp,-64 ; align stack frame + mov QWORD[((128+0))+rsp],rdi ; save ctx, 1st arg + mov QWORD[((128+8))+rsp],rsi ; save inp, 2nd arh + mov QWORD[((128+16))+rsp],rdx ; save end pointer, "3rd" arg + mov QWORD[152+rsp],rax ; save copy of %rsp $L$prologue: @@ -68,7 +68,7 @@ $L$loop: mov rdi,rbx lea rbp,[K512] - xor rdi,rcx + xor rdi,rcx ; magic mov r12,QWORD[rsi] mov r13,r8 mov r14,rax @@ -78,38 +78,38 @@ xor r13,r8 ror r14,5 - xor r15,r10 + xor r15,r10 ; f^g mov QWORD[rsp],r12 xor r14,rax - and r15,r8 + and r15,r8 ; (f^g)&e ror r13,4 - add r12,r11 - xor r15,r10 + add r12,r11 ; T1+=h + xor r15,r10 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r8 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rax - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rax - xor r15,rbx - ror r13,14 + xor r15,rbx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r11,rbx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r11,rdi - add rdx,r12 - add r11,r12 + xor r11,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rdx,r12 ; d+=T1 + add r11,r12 ; h+=T1 - lea rbp,[8+rbp] - add r11,r14 + lea rbp,[8+rbp] ; round++ + add r11,r14 ; h+=Sigma0(a) mov r12,QWORD[8+rsi] mov r13,rdx mov r14,r11 @@ -119,38 +119,38 @@ xor r13,rdx ror r14,5 - xor rdi,r9 + xor rdi,r9 ; f^g mov QWORD[8+rsp],r12 xor r14,r11 - and rdi,rdx + and rdi,rdx ; (f^g)&e ror r13,4 - add r12,r10 - xor rdi,r9 + add r12,r10 ; T1+=h + xor rdi,r9 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rdx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r11 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r11 - xor rdi,rax - ror r13,14 + xor rdi,rax ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r10,rax and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r10,r15 - add rcx,r12 - add r10,r12 + xor r10,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rcx,r12 ; d+=T1 + add r10,r12 ; h+=T1 - lea rbp,[24+rbp] - add r10,r14 + lea rbp,[24+rbp] ; round++ + add r10,r14 ; h+=Sigma0(a) mov r12,QWORD[16+rsi] mov r13,rcx mov r14,r10 @@ -160,38 +160,38 @@ xor r13,rcx ror r14,5 - xor r15,r8 + xor r15,r8 ; f^g mov QWORD[16+rsp],r12 xor r14,r10 - and r15,rcx + and r15,rcx ; (f^g)&e ror r13,4 - add r12,r9 - xor r15,r8 + add r12,r9 ; T1+=h + xor r15,r8 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rcx - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r10 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r10 - xor r15,r11 - ror r13,14 + xor r15,r11 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r9,r11 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r9,rdi - add rbx,r12 - add r9,r12 + xor r9,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rbx,r12 ; d+=T1 + add r9,r12 ; h+=T1 - lea rbp,[8+rbp] - add r9,r14 + lea rbp,[8+rbp] ; round++ + add r9,r14 ; h+=Sigma0(a) mov r12,QWORD[24+rsi] mov r13,rbx mov r14,r9 @@ -201,38 +201,38 @@ xor r13,rbx ror r14,5 - xor rdi,rdx + xor rdi,rdx ; f^g mov QWORD[24+rsp],r12 xor r14,r9 - and rdi,rbx + and rdi,rbx ; (f^g)&e ror r13,4 - add r12,r8 - xor rdi,rdx + add r12,r8 ; T1+=h + xor rdi,rdx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rbx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r9 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r9 - xor rdi,r10 - ror r13,14 + xor rdi,r10 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r8,r10 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r8,r15 - add rax,r12 - add r8,r12 + xor r8,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rax,r12 ; d+=T1 + add r8,r12 ; h+=T1 - lea rbp,[24+rbp] - add r8,r14 + lea rbp,[24+rbp] ; round++ + add r8,r14 ; h+=Sigma0(a) mov r12,QWORD[32+rsi] mov r13,rax mov r14,r8 @@ -242,38 +242,38 @@ xor r13,rax ror r14,5 - xor r15,rcx + xor r15,rcx ; f^g mov QWORD[32+rsp],r12 xor r14,r8 - and r15,rax + and r15,rax ; (f^g)&e ror r13,4 - add r12,rdx - xor r15,rcx + add r12,rdx ; T1+=h + xor r15,rcx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rax - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r8 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r8 - xor r15,r9 - ror r13,14 + xor r15,r9 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rdx,r9 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rdx,rdi - add r11,r12 - add rdx,r12 + xor rdx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11,r12 ; d+=T1 + add rdx,r12 ; h+=T1 - lea rbp,[8+rbp] - add rdx,r14 + lea rbp,[8+rbp] ; round++ + add rdx,r14 ; h+=Sigma0(a) mov r12,QWORD[40+rsi] mov r13,r11 mov r14,rdx @@ -283,38 +283,38 @@ xor r13,r11 ror r14,5 - xor rdi,rbx + xor rdi,rbx ; f^g mov QWORD[40+rsp],r12 xor r14,rdx - and rdi,r11 + and rdi,r11 ; (f^g)&e ror r13,4 - add r12,rcx - xor rdi,rbx + add r12,rcx ; T1+=h + xor rdi,rbx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r11 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rdx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rdx - xor rdi,r8 - ror r13,14 + xor rdi,r8 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rcx,r8 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rcx,r15 - add r10,r12 - add rcx,r12 + xor rcx,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10,r12 ; d+=T1 + add rcx,r12 ; h+=T1 - lea rbp,[24+rbp] - add rcx,r14 + lea rbp,[24+rbp] ; round++ + add rcx,r14 ; h+=Sigma0(a) mov r12,QWORD[48+rsi] mov r13,r10 mov r14,rcx @@ -324,38 +324,38 @@ xor r13,r10 ror r14,5 - xor r15,rax + xor r15,rax ; f^g mov QWORD[48+rsp],r12 xor r14,rcx - and r15,r10 + and r15,r10 ; (f^g)&e ror r13,4 - add r12,rbx - xor r15,rax + add r12,rbx ; T1+=h + xor r15,rax ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r10 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rcx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rcx - xor r15,rdx - ror r13,14 + xor r15,rdx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rbx,rdx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rbx,rdi - add r9,r12 - add rbx,r12 + xor rbx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9,r12 ; d+=T1 + add rbx,r12 ; h+=T1 - lea rbp,[8+rbp] - add rbx,r14 + lea rbp,[8+rbp] ; round++ + add rbx,r14 ; h+=Sigma0(a) mov r12,QWORD[56+rsi] mov r13,r9 mov r14,rbx @@ -365,38 +365,38 @@ xor r13,r9 ror r14,5 - xor rdi,r11 + xor rdi,r11 ; f^g mov QWORD[56+rsp],r12 xor r14,rbx - and rdi,r9 + and rdi,r9 ; (f^g)&e ror r13,4 - add r12,rax - xor rdi,r11 + add r12,rax ; T1+=h + xor rdi,r11 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r9 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rbx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rbx - xor rdi,rcx - ror r13,14 + xor rdi,rcx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rax,rcx and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rax,r15 - add r8,r12 - add rax,r12 + xor rax,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8,r12 ; d+=T1 + add rax,r12 ; h+=T1 - lea rbp,[24+rbp] - add rax,r14 + lea rbp,[24+rbp] ; round++ + add rax,r14 ; h+=Sigma0(a) mov r12,QWORD[64+rsi] mov r13,r8 mov r14,rax @@ -406,38 +406,38 @@ xor r13,r8 ror r14,5 - xor r15,r10 + xor r15,r10 ; f^g mov QWORD[64+rsp],r12 xor r14,rax - and r15,r8 + and r15,r8 ; (f^g)&e ror r13,4 - add r12,r11 - xor r15,r10 + add r12,r11 ; T1+=h + xor r15,r10 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r8 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rax - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rax - xor r15,rbx - ror r13,14 + xor r15,rbx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r11,rbx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r11,rdi - add rdx,r12 - add r11,r12 + xor r11,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rdx,r12 ; d+=T1 + add r11,r12 ; h+=T1 - lea rbp,[8+rbp] - add r11,r14 + lea rbp,[8+rbp] ; round++ + add r11,r14 ; h+=Sigma0(a) mov r12,QWORD[72+rsi] mov r13,rdx mov r14,r11 @@ -447,38 +447,38 @@ xor r13,rdx ror r14,5 - xor rdi,r9 + xor rdi,r9 ; f^g mov QWORD[72+rsp],r12 xor r14,r11 - and rdi,rdx + and rdi,rdx ; (f^g)&e ror r13,4 - add r12,r10 - xor rdi,r9 + add r12,r10 ; T1+=h + xor rdi,r9 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rdx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r11 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r11 - xor rdi,rax - ror r13,14 + xor rdi,rax ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r10,rax and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r10,r15 - add rcx,r12 - add r10,r12 + xor r10,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rcx,r12 ; d+=T1 + add r10,r12 ; h+=T1 - lea rbp,[24+rbp] - add r10,r14 + lea rbp,[24+rbp] ; round++ + add r10,r14 ; h+=Sigma0(a) mov r12,QWORD[80+rsi] mov r13,rcx mov r14,r10 @@ -488,38 +488,38 @@ xor r13,rcx ror r14,5 - xor r15,r8 + xor r15,r8 ; f^g mov QWORD[80+rsp],r12 xor r14,r10 - and r15,rcx + and r15,rcx ; (f^g)&e ror r13,4 - add r12,r9 - xor r15,r8 + add r12,r9 ; T1+=h + xor r15,r8 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rcx - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r10 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r10 - xor r15,r11 - ror r13,14 + xor r15,r11 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r9,r11 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r9,rdi - add rbx,r12 - add r9,r12 + xor r9,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rbx,r12 ; d+=T1 + add r9,r12 ; h+=T1 - lea rbp,[8+rbp] - add r9,r14 + lea rbp,[8+rbp] ; round++ + add r9,r14 ; h+=Sigma0(a) mov r12,QWORD[88+rsi] mov r13,rbx mov r14,r9 @@ -529,38 +529,38 @@ xor r13,rbx ror r14,5 - xor rdi,rdx + xor rdi,rdx ; f^g mov QWORD[88+rsp],r12 xor r14,r9 - and rdi,rbx + and rdi,rbx ; (f^g)&e ror r13,4 - add r12,r8 - xor rdi,rdx + add r12,r8 ; T1+=h + xor rdi,rdx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rbx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r9 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r9 - xor rdi,r10 - ror r13,14 + xor rdi,r10 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r8,r10 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r8,r15 - add rax,r12 - add r8,r12 + xor r8,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rax,r12 ; d+=T1 + add r8,r12 ; h+=T1 - lea rbp,[24+rbp] - add r8,r14 + lea rbp,[24+rbp] ; round++ + add r8,r14 ; h+=Sigma0(a) mov r12,QWORD[96+rsi] mov r13,rax mov r14,r8 @@ -570,38 +570,38 @@ xor r13,rax ror r14,5 - xor r15,rcx + xor r15,rcx ; f^g mov QWORD[96+rsp],r12 xor r14,r8 - and r15,rax + and r15,rax ; (f^g)&e ror r13,4 - add r12,rdx - xor r15,rcx + add r12,rdx ; T1+=h + xor r15,rcx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rax - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r8 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r8 - xor r15,r9 - ror r13,14 + xor r15,r9 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rdx,r9 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rdx,rdi - add r11,r12 - add rdx,r12 + xor rdx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11,r12 ; d+=T1 + add rdx,r12 ; h+=T1 - lea rbp,[8+rbp] - add rdx,r14 + lea rbp,[8+rbp] ; round++ + add rdx,r14 ; h+=Sigma0(a) mov r12,QWORD[104+rsi] mov r13,r11 mov r14,rdx @@ -611,38 +611,38 @@ xor r13,r11 ror r14,5 - xor rdi,rbx + xor rdi,rbx ; f^g mov QWORD[104+rsp],r12 xor r14,rdx - and rdi,r11 + and rdi,r11 ; (f^g)&e ror r13,4 - add r12,rcx - xor rdi,rbx + add r12,rcx ; T1+=h + xor rdi,rbx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r11 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rdx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rdx - xor rdi,r8 - ror r13,14 + xor rdi,r8 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rcx,r8 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rcx,r15 - add r10,r12 - add rcx,r12 + xor rcx,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10,r12 ; d+=T1 + add rcx,r12 ; h+=T1 - lea rbp,[24+rbp] - add rcx,r14 + lea rbp,[24+rbp] ; round++ + add rcx,r14 ; h+=Sigma0(a) mov r12,QWORD[112+rsi] mov r13,r10 mov r14,rcx @@ -652,38 +652,38 @@ xor r13,r10 ror r14,5 - xor r15,rax + xor r15,rax ; f^g mov QWORD[112+rsp],r12 xor r14,rcx - and r15,r10 + and r15,r10 ; (f^g)&e ror r13,4 - add r12,rbx - xor r15,rax + add r12,rbx ; T1+=h + xor r15,rax ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r10 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rcx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rcx - xor r15,rdx - ror r13,14 + xor r15,rdx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rbx,rdx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rbx,rdi - add r9,r12 - add rbx,r12 + xor rbx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9,r12 ; d+=T1 + add rbx,r12 ; h+=T1 - lea rbp,[8+rbp] - add rbx,r14 + lea rbp,[8+rbp] ; round++ + add rbx,r14 ; h+=Sigma0(a) mov r12,QWORD[120+rsi] mov r13,r9 mov r14,rbx @@ -693,37 +693,37 @@ xor r13,r9 ror r14,5 - xor rdi,r11 + xor rdi,r11 ; f^g mov QWORD[120+rsp],r12 xor r14,rbx - and rdi,r9 + and rdi,r9 ; (f^g)&e ror r13,4 - add r12,rax - xor rdi,r11 + add r12,rax ; T1+=h + xor rdi,r11 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r9 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rbx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rbx - xor rdi,rcx - ror r13,14 + xor rdi,rcx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rax,rcx and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rax,r15 - add r8,r12 - add rax,r12 + xor rax,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8,r12 ; d+=T1 + add rax,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ jmp NEAR $L$rounds_16_xx ALIGN 16 $L$rounds_16_xx: @@ -732,7 +732,7 @@ mov r12,r13 ror r13,7 - add rax,r14 + add rax,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -743,8 +743,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[72+rsp] add r12,QWORD[rsp] @@ -756,43 +756,43 @@ xor r13,r8 ror r14,5 - xor r15,r10 + xor r15,r10 ; f^g mov QWORD[rsp],r12 xor r14,rax - and r15,r8 + and r15,r8 ; (f^g)&e ror r13,4 - add r12,r11 - xor r15,r10 + add r12,r11 ; T1+=h + xor r15,r10 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r8 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rax - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rax - xor r15,rbx - ror r13,14 + xor r15,rbx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r11,rbx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r11,rdi - add rdx,r12 - add r11,r12 + xor r11,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rdx,r12 ; d+=T1 + add r11,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[16+rsp] mov rdi,QWORD[120+rsp] mov r12,r13 ror r13,7 - add r11,r14 + add r11,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -803,8 +803,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[80+rsp] add r12,QWORD[8+rsp] @@ -816,43 +816,43 @@ xor r13,rdx ror r14,5 - xor rdi,r9 + xor rdi,r9 ; f^g mov QWORD[8+rsp],r12 xor r14,r11 - and rdi,rdx + and rdi,rdx ; (f^g)&e ror r13,4 - add r12,r10 - xor rdi,r9 + add r12,r10 ; T1+=h + xor rdi,r9 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rdx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r11 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r11 - xor rdi,rax - ror r13,14 + xor rdi,rax ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r10,rax and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r10,r15 - add rcx,r12 - add r10,r12 + xor r10,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rcx,r12 ; d+=T1 + add r10,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[24+rsp] mov r15,QWORD[rsp] mov r12,r13 ror r13,7 - add r10,r14 + add r10,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -863,8 +863,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[88+rsp] add r12,QWORD[16+rsp] @@ -876,43 +876,43 @@ xor r13,rcx ror r14,5 - xor r15,r8 + xor r15,r8 ; f^g mov QWORD[16+rsp],r12 xor r14,r10 - and r15,rcx + and r15,rcx ; (f^g)&e ror r13,4 - add r12,r9 - xor r15,r8 + add r12,r9 ; T1+=h + xor r15,r8 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rcx - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r10 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r10 - xor r15,r11 - ror r13,14 + xor r15,r11 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r9,r11 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r9,rdi - add rbx,r12 - add r9,r12 + xor r9,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rbx,r12 ; d+=T1 + add r9,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[32+rsp] mov rdi,QWORD[8+rsp] mov r12,r13 ror r13,7 - add r9,r14 + add r9,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -923,8 +923,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[96+rsp] add r12,QWORD[24+rsp] @@ -936,43 +936,43 @@ xor r13,rbx ror r14,5 - xor rdi,rdx + xor rdi,rdx ; f^g mov QWORD[24+rsp],r12 xor r14,r9 - and rdi,rbx + and rdi,rbx ; (f^g)&e ror r13,4 - add r12,r8 - xor rdi,rdx + add r12,r8 ; T1+=h + xor rdi,rdx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rbx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r9 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r9 - xor rdi,r10 - ror r13,14 + xor rdi,r10 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r8,r10 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r8,r15 - add rax,r12 - add r8,r12 + xor r8,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rax,r12 ; d+=T1 + add r8,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[40+rsp] mov r15,QWORD[16+rsp] mov r12,r13 ror r13,7 - add r8,r14 + add r8,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -983,8 +983,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[104+rsp] add r12,QWORD[32+rsp] @@ -996,43 +996,43 @@ xor r13,rax ror r14,5 - xor r15,rcx + xor r15,rcx ; f^g mov QWORD[32+rsp],r12 xor r14,r8 - and r15,rax + and r15,rax ; (f^g)&e ror r13,4 - add r12,rdx - xor r15,rcx + add r12,rdx ; T1+=h + xor r15,rcx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rax - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r8 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r8 - xor r15,r9 - ror r13,14 + xor r15,r9 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rdx,r9 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rdx,rdi - add r11,r12 - add rdx,r12 + xor rdx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11,r12 ; d+=T1 + add rdx,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[48+rsp] mov rdi,QWORD[24+rsp] mov r12,r13 ror r13,7 - add rdx,r14 + add rdx,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -1043,8 +1043,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[112+rsp] add r12,QWORD[40+rsp] @@ -1056,43 +1056,43 @@ xor r13,r11 ror r14,5 - xor rdi,rbx + xor rdi,rbx ; f^g mov QWORD[40+rsp],r12 xor r14,rdx - and rdi,r11 + and rdi,r11 ; (f^g)&e ror r13,4 - add r12,rcx - xor rdi,rbx + add r12,rcx ; T1+=h + xor rdi,rbx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r11 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rdx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rdx - xor rdi,r8 - ror r13,14 + xor rdi,r8 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rcx,r8 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rcx,r15 - add r10,r12 - add rcx,r12 + xor rcx,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10,r12 ; d+=T1 + add rcx,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[56+rsp] mov r15,QWORD[32+rsp] mov r12,r13 ror r13,7 - add rcx,r14 + add rcx,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -1103,8 +1103,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[120+rsp] add r12,QWORD[48+rsp] @@ -1116,43 +1116,43 @@ xor r13,r10 ror r14,5 - xor r15,rax + xor r15,rax ; f^g mov QWORD[48+rsp],r12 xor r14,rcx - and r15,r10 + and r15,r10 ; (f^g)&e ror r13,4 - add r12,rbx - xor r15,rax + add r12,rbx ; T1+=h + xor r15,rax ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r10 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rcx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rcx - xor r15,rdx - ror r13,14 + xor r15,rdx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rbx,rdx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rbx,rdi - add r9,r12 - add rbx,r12 + xor rbx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9,r12 ; d+=T1 + add rbx,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[64+rsp] mov rdi,QWORD[40+rsp] mov r12,r13 ror r13,7 - add rbx,r14 + add rbx,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -1163,8 +1163,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[rsp] add r12,QWORD[56+rsp] @@ -1176,43 +1176,43 @@ xor r13,r9 ror r14,5 - xor rdi,r11 + xor rdi,r11 ; f^g mov QWORD[56+rsp],r12 xor r14,rbx - and rdi,r9 + and rdi,r9 ; (f^g)&e ror r13,4 - add r12,rax - xor rdi,r11 + add r12,rax ; T1+=h + xor rdi,r11 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r9 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rbx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rbx - xor rdi,rcx - ror r13,14 + xor rdi,rcx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rax,rcx and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rax,r15 - add r8,r12 - add rax,r12 + xor rax,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8,r12 ; d+=T1 + add rax,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[72+rsp] mov r15,QWORD[48+rsp] mov r12,r13 ror r13,7 - add rax,r14 + add rax,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -1223,8 +1223,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[8+rsp] add r12,QWORD[64+rsp] @@ -1236,43 +1236,43 @@ xor r13,r8 ror r14,5 - xor r15,r10 + xor r15,r10 ; f^g mov QWORD[64+rsp],r12 xor r14,rax - and r15,r8 + and r15,r8 ; (f^g)&e ror r13,4 - add r12,r11 - xor r15,r10 + add r12,r11 ; T1+=h + xor r15,r10 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r8 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rax - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rax - xor r15,rbx - ror r13,14 + xor r15,rbx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r11,rbx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r11,rdi - add rdx,r12 - add r11,r12 + xor r11,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rdx,r12 ; d+=T1 + add r11,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[80+rsp] mov rdi,QWORD[56+rsp] mov r12,r13 ror r13,7 - add r11,r14 + add r11,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -1283,8 +1283,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[16+rsp] add r12,QWORD[72+rsp] @@ -1296,43 +1296,43 @@ xor r13,rdx ror r14,5 - xor rdi,r9 + xor rdi,r9 ; f^g mov QWORD[72+rsp],r12 xor r14,r11 - and rdi,rdx + and rdi,rdx ; (f^g)&e ror r13,4 - add r12,r10 - xor rdi,r9 + add r12,r10 ; T1+=h + xor rdi,r9 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rdx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r11 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r11 - xor rdi,rax - ror r13,14 + xor rdi,rax ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r10,rax and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r10,r15 - add rcx,r12 - add r10,r12 + xor r10,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rcx,r12 ; d+=T1 + add r10,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[88+rsp] mov r15,QWORD[64+rsp] mov r12,r13 ror r13,7 - add r10,r14 + add r10,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -1343,8 +1343,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[24+rsp] add r12,QWORD[80+rsp] @@ -1356,43 +1356,43 @@ xor r13,rcx ror r14,5 - xor r15,r8 + xor r15,r8 ; f^g mov QWORD[80+rsp],r12 xor r14,r10 - and r15,rcx + and r15,rcx ; (f^g)&e ror r13,4 - add r12,r9 - xor r15,r8 + add r12,r9 ; T1+=h + xor r15,r8 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rcx - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r10 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r10 - xor r15,r11 - ror r13,14 + xor r15,r11 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r9,r11 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r9,rdi - add rbx,r12 - add r9,r12 + xor r9,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rbx,r12 ; d+=T1 + add r9,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[96+rsp] mov rdi,QWORD[72+rsp] mov r12,r13 ror r13,7 - add r9,r14 + add r9,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -1403,8 +1403,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[32+rsp] add r12,QWORD[88+rsp] @@ -1416,43 +1416,43 @@ xor r13,rbx ror r14,5 - xor rdi,rdx + xor rdi,rdx ; f^g mov QWORD[88+rsp],r12 xor r14,r9 - and rdi,rbx + and rdi,rbx ; (f^g)&e ror r13,4 - add r12,r8 - xor rdi,rdx + add r12,r8 ; T1+=h + xor rdi,rdx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rbx - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,r9 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r9 - xor rdi,r10 - ror r13,14 + xor rdi,r10 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov r8,r10 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor r8,r15 - add rax,r12 - add r8,r12 + xor r8,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add rax,r12 ; d+=T1 + add r8,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[104+rsp] mov r15,QWORD[80+rsp] mov r12,r13 ror r13,7 - add r8,r14 + add r8,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -1463,8 +1463,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[40+rsp] add r12,QWORD[96+rsp] @@ -1476,43 +1476,43 @@ xor r13,rax ror r14,5 - xor r15,rcx + xor r15,rcx ; f^g mov QWORD[96+rsp],r12 xor r14,r8 - and r15,rax + and r15,rax ; (f^g)&e ror r13,4 - add r12,rdx - xor r15,rcx + add r12,rdx ; T1+=h + xor r15,rcx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,rax - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,r8 - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,r8 - xor r15,r9 - ror r13,14 + xor r15,r9 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rdx,r9 and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rdx,rdi - add r11,r12 - add rdx,r12 + xor rdx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r11,r12 ; d+=T1 + add rdx,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[112+rsp] mov rdi,QWORD[88+rsp] mov r12,r13 ror r13,7 - add rdx,r14 + add rdx,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -1523,8 +1523,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[48+rsp] add r12,QWORD[104+rsp] @@ -1536,43 +1536,43 @@ xor r13,r11 ror r14,5 - xor rdi,rbx + xor rdi,rbx ; f^g mov QWORD[104+rsp],r12 xor r14,rdx - and rdi,r11 + and rdi,r11 ; (f^g)&e ror r13,4 - add r12,rcx - xor rdi,rbx + add r12,rcx ; T1+=h + xor rdi,rbx ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r11 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rdx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rdx - xor rdi,r8 - ror r13,14 + xor rdi,r8 ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rcx,r8 and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rcx,r15 - add r10,r12 - add rcx,r12 + xor rcx,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r10,r12 ; d+=T1 + add rcx,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ mov r13,QWORD[120+rsp] mov r15,QWORD[96+rsp] mov r12,r13 ror r13,7 - add rcx,r14 + add rcx,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,r15 ror r15,42 @@ -1583,8 +1583,8 @@ shr r14,6 ror r15,19 - xor r12,r13 - xor r15,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor r15,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[56+rsp] add r12,QWORD[112+rsp] @@ -1596,43 +1596,43 @@ xor r13,r10 ror r14,5 - xor r15,rax + xor r15,rax ; f^g mov QWORD[112+rsp],r12 xor r14,rcx - and r15,r10 + and r15,r10 ; (f^g)&e ror r13,4 - add r12,rbx - xor r15,rax + add r12,rbx ; T1+=h + xor r15,rax ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r10 - add r12,r15 + add r12,r15 ; T1+=Ch(e,f,g) mov r15,rcx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rcx - xor r15,rdx - ror r13,14 + xor r15,rdx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rbx,rdx and rdi,r15 - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rbx,rdi - add r9,r12 - add rbx,r12 + xor rbx,rdi ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r9,r12 ; d+=T1 + add rbx,r12 ; h+=T1 - lea rbp,[8+rbp] + lea rbp,[8+rbp] ; round++ mov r13,QWORD[rsp] mov rdi,QWORD[104+rsp] mov r12,r13 ror r13,7 - add rbx,r14 + add rbx,r14 ; modulo-scheduled h+=Sigma0(a) mov r14,rdi ror rdi,42 @@ -1643,8 +1643,8 @@ shr r14,6 ror rdi,19 - xor r12,r13 - xor rdi,r14 + xor r12,r13 ; sigma0(X[(i+1)&0xf]) + xor rdi,r14 ; sigma1(X[(i+14)&0xf]) add r12,QWORD[64+rsp] add r12,QWORD[120+rsp] @@ -1656,42 +1656,42 @@ xor r13,r9 ror r14,5 - xor rdi,r11 + xor rdi,r11 ; f^g mov QWORD[120+rsp],r12 xor r14,rbx - and rdi,r9 + and rdi,r9 ; (f^g)&e ror r13,4 - add r12,rax - xor rdi,r11 + add r12,rax ; T1+=h + xor rdi,r11 ; Ch(e,f,g)=((f^g)&e)^g ror r14,6 xor r13,r9 - add r12,rdi + add r12,rdi ; T1+=Ch(e,f,g) mov rdi,rbx - add r12,QWORD[rbp] + add r12,QWORD[rbp] ; T1+=K[round] xor r14,rbx - xor rdi,rcx - ror r13,14 + xor rdi,rcx ; a^b, b^c in next round + ror r13,14 ; Sigma1(e) mov rax,rcx and r15,rdi - ror r14,28 - add r12,r13 + ror r14,28 ; Sigma0(a) + add r12,r13 ; T1+=Sigma1(e) - xor rax,r15 - add r8,r12 - add rax,r12 + xor rax,r15 ; h=Maj(a,b,c)=Ch(a^b,c,b) + add r8,r12 ; d+=T1 + add rax,r12 ; h+=T1 - lea rbp,[24+rbp] + lea rbp,[24+rbp] ; round++ cmp BYTE[7+rbp],0 jnz NEAR $L$rounds_16_xx mov rdi,QWORD[((128+0))+rsp] - add rax,r14 + add rax,r14 ; modulo-scheduled h+=Sigma0(a) lea rsi,[128+rsi] add rax,QWORD[rdi] @@ -1846,7 +1846,7 @@ _CET_ENDBR - mov rax,rsp + mov rax,rsp ; copy %rsp push rbx @@ -1860,14 +1860,14 @@ push r15 - shl rdx,4 + shl rdx,4 ; num*16 sub rsp,256 - lea rdx,[rdx*8+rsi] - and rsp,-64 - mov QWORD[((128+0))+rsp],rdi - mov QWORD[((128+8))+rsp],rsi - mov QWORD[((128+16))+rsp],rdx - mov QWORD[152+rsp],rax + lea rdx,[rdx*8+rsi] ; inp+num*16*8 + and rsp,-64 ; align stack frame + mov QWORD[((128+0))+rsp],rdi ; save ctx, 1st arg + mov QWORD[((128+8))+rsp],rsi ; save inp, 2nd arh + mov QWORD[((128+16))+rsp],rdx ; save end pointer, "3rd" arg + mov QWORD[152+rsp],rax ; save copy of %rsp movaps XMMWORD[(128+32)+rsp],xmm6 movaps XMMWORD[(128+48)+rsp],xmm7 @@ -1891,7 +1891,7 @@ $L$loop_avx: vmovdqa xmm11,XMMWORD[((K512+1280))] vmovdqu xmm0,XMMWORD[rsi] - lea rbp,[((K512+128))] + lea rbp,[((K512+128))] ; size optimization vmovdqu xmm1,XMMWORD[16+rsi] vmovdqu xmm2,XMMWORD[32+rsi] vpshufb xmm0,xmm0,xmm11 @@ -1924,7 +1924,7 @@ vmovdqa XMMWORD[80+rsp],xmm9 mov rdi,rbx vmovdqa XMMWORD[96+rsp],xmm10 - xor rdi,rcx + xor rdi,rcx ; magic vmovdqa XMMWORD[112+rsp],xmm11 mov r13,r8 jmp NEAR $L$avx_00_47 @@ -3035,25 +3035,25 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HanderlData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$in_prologue - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$in_prologue - mov rsi,rax - mov rax,QWORD[((128+24))+rax] + mov rsi,rax ; put aside Rsp + mov rax,QWORD[((128+24))+rax] ; pull 152(%rsp) mov rbx,QWORD[((-8))+rax] mov rbp,QWORD[((-16))+rax] @@ -3061,49 +3061,49 @@ mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R15 lea r10,[$L$epilogue] cmp rbx,r10 - jb NEAR $L$in_prologue + jb NEAR $L$in_prologue ; non-AVX code - lea rsi,[((128+32))+rsi] - lea rdi,[512+r8] + lea rsi,[((128+32))+rsi] ; Xmm6- save area + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,12 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq $L$in_prologue: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -3129,11 +3129,11 @@ $L$SEH_info_sha512_block_data_order_nohw: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$prologue wrt ..imagebase,$L$epilogue wrt ..imagebase + DD $L$prologue wrt ..imagebase,$L$epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_sha512_block_data_order_avx: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase + DD $L$prologue_avx wrt ..imagebase,$L$epilogue_avx wrt ..imagebase ; HandlerData[] %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/bcm/vpaes-x86_64-apple.S b/gen/bcm/vpaes-x86_64-apple.S index ef76230..38cfd00 100644 --- a/gen/bcm/vpaes-x86_64-apple.S +++ b/gen/bcm/vpaes-x86_64-apple.S
@@ -6,21 +6,21 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) .text - - - - - - - - - - - - - - - +// +// _aes_encrypt_core +// +// AES-encrypt %xmm0. +// +// Inputs: +// %xmm0 = input +// %xmm9-%xmm15 as in _vpaes_preheat +// (%rdx) = scheduled keys +// +// Output in %xmm0 +// Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax +// Preserves %xmm6 - %xmm8 so you get some local vectors +// +// .p2align 4 _vpaes_encrypt_core: @@ -29,13 +29,13 @@ movq $16,%r11 movl 240(%rdx),%eax movdqa %xmm9,%xmm1 - movdqa L$k_ipt(%rip),%xmm2 + movdqa L$k_ipt(%rip),%xmm2 // iptlo pandn %xmm0,%xmm1 - movdqu (%r9),%xmm5 + movdqu (%r9),%xmm5 // round0 key psrld $4,%xmm1 pand %xmm9,%xmm0 pshufb %xmm0,%xmm2 - movdqa L$k_ipt+16(%rip),%xmm0 + movdqa L$k_ipt+16(%rip),%xmm0 // ipthi pshufb %xmm1,%xmm0 pxor %xmm5,%xmm2 addq $16,%r9 @@ -45,98 +45,98 @@ .p2align 4 L$enc_loop: - - movdqa %xmm13,%xmm4 - movdqa %xmm12,%xmm0 - pshufb %xmm2,%xmm4 - pshufb %xmm3,%xmm0 - pxor %xmm5,%xmm4 - movdqa %xmm15,%xmm5 - pxor %xmm4,%xmm0 - movdqa -64(%r11,%r10,1),%xmm1 - pshufb %xmm2,%xmm5 - movdqa (%r11,%r10,1),%xmm4 - movdqa %xmm14,%xmm2 - pshufb %xmm3,%xmm2 - movdqa %xmm0,%xmm3 - pxor %xmm5,%xmm2 - pshufb %xmm1,%xmm0 - addq $16,%r9 - pxor %xmm2,%xmm0 - pshufb %xmm4,%xmm3 - addq $16,%r11 - pxor %xmm0,%xmm3 - pshufb %xmm1,%xmm0 - andq $0x30,%r11 - subq $1,%rax - pxor %xmm3,%xmm0 +// middle of middle round + movdqa %xmm13,%xmm4 // 4 : sb1u + movdqa %xmm12,%xmm0 // 0 : sb1t + pshufb %xmm2,%xmm4 // 4 = sb1u + pshufb %xmm3,%xmm0 // 0 = sb1t + pxor %xmm5,%xmm4 // 4 = sb1u + k + movdqa %xmm15,%xmm5 // 4 : sb2u + pxor %xmm4,%xmm0 // 0 = A + movdqa -64(%r11,%r10,1),%xmm1 // .Lk_mc_forward[] + pshufb %xmm2,%xmm5 // 4 = sb2u + movdqa (%r11,%r10,1),%xmm4 // .Lk_mc_backward[] + movdqa %xmm14,%xmm2 // 2 : sb2t + pshufb %xmm3,%xmm2 // 2 = sb2t + movdqa %xmm0,%xmm3 // 3 = A + pxor %xmm5,%xmm2 // 2 = 2A + pshufb %xmm1,%xmm0 // 0 = B + addq $16,%r9 // next key + pxor %xmm2,%xmm0 // 0 = 2A+B + pshufb %xmm4,%xmm3 // 3 = D + addq $16,%r11 // next mc + pxor %xmm0,%xmm3 // 3 = 2A+B+D + pshufb %xmm1,%xmm0 // 0 = 2B+C + andq $0x30,%r11 // ... mod 4 + subq $1,%rax // nr-- + pxor %xmm3,%xmm0 // 0 = 2A+3B+C+D L$enc_entry: - - movdqa %xmm9,%xmm1 - movdqa %xmm11,%xmm5 - pandn %xmm0,%xmm1 - psrld $4,%xmm1 - pand %xmm9,%xmm0 - pshufb %xmm0,%xmm5 - movdqa %xmm10,%xmm3 - pxor %xmm1,%xmm0 - pshufb %xmm1,%xmm3 - movdqa %xmm10,%xmm4 - pxor %xmm5,%xmm3 - pshufb %xmm0,%xmm4 - movdqa %xmm10,%xmm2 - pxor %xmm5,%xmm4 - pshufb %xmm3,%xmm2 - movdqa %xmm10,%xmm3 - pxor %xmm0,%xmm2 - pshufb %xmm4,%xmm3 +// top of round + movdqa %xmm9,%xmm1 // 1 : i + movdqa %xmm11,%xmm5 // 2 : a/k + pandn %xmm0,%xmm1 // 1 = i<<4 + psrld $4,%xmm1 // 1 = i + pand %xmm9,%xmm0 // 0 = k + pshufb %xmm0,%xmm5 // 2 = a/k + movdqa %xmm10,%xmm3 // 3 : 1/i + pxor %xmm1,%xmm0 // 0 = j + pshufb %xmm1,%xmm3 // 3 = 1/i + movdqa %xmm10,%xmm4 // 4 : 1/j + pxor %xmm5,%xmm3 // 3 = iak = 1/i + a/k + pshufb %xmm0,%xmm4 // 4 = 1/j + movdqa %xmm10,%xmm2 // 2 : 1/iak + pxor %xmm5,%xmm4 // 4 = jak = 1/j + a/k + pshufb %xmm3,%xmm2 // 2 = 1/iak + movdqa %xmm10,%xmm3 // 3 : 1/jak + pxor %xmm0,%xmm2 // 2 = io + pshufb %xmm4,%xmm3 // 3 = 1/jak movdqu (%r9),%xmm5 - pxor %xmm1,%xmm3 + pxor %xmm1,%xmm3 // 3 = jo jnz L$enc_loop - - movdqa -96(%r10),%xmm4 - movdqa -80(%r10),%xmm0 - pshufb %xmm2,%xmm4 - pxor %xmm5,%xmm4 - pshufb %xmm3,%xmm0 - movdqa 64(%r11,%r10,1),%xmm1 - pxor %xmm4,%xmm0 +// middle of last round + movdqa -96(%r10),%xmm4 // 3 : sbou .Lk_sbo + movdqa -80(%r10),%xmm0 // 0 : sbot .Lk_sbo+16 + pshufb %xmm2,%xmm4 // 4 = sbou + pxor %xmm5,%xmm4 // 4 = sb1u + k + pshufb %xmm3,%xmm0 // 0 = sb1t + movdqa 64(%r11,%r10,1),%xmm1 // .Lk_sr[] + pxor %xmm4,%xmm0 // 0 = A pshufb %xmm1,%xmm0 ret - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// +// _aes_encrypt_core_2x +// +// AES-encrypt %xmm0 and %xmm6 in parallel. +// +// Inputs: +// %xmm0 and %xmm6 = input +// %xmm9 and %xmm10 as in _vpaes_preheat +// (%rdx) = scheduled keys +// +// Output in %xmm0 and %xmm6 +// Clobbers %xmm1-%xmm5, %xmm7, %xmm8, %xmm11-%xmm13, %r9, %r10, %r11, %rax +// Preserves %xmm14 and %xmm15 +// +// This function stitches two parallel instances of _vpaes_encrypt_core. x86_64 +// provides 16 XMM registers. _vpaes_encrypt_core computes over six registers +// (%xmm0-%xmm5) and additionally uses seven registers with preloaded constants +// from _vpaes_preheat (%xmm9-%xmm15). This does not quite fit two instances, +// so we spill some of %xmm9 through %xmm15 back to memory. We keep %xmm9 and +// %xmm10 in registers as these values are used several times in a row. The +// remainder are read once per round and are spilled to memory. This leaves two +// registers preserved for the caller. +// +// Thus, of the two _vpaes_encrypt_core instances, the first uses (%xmm0-%xmm5) +// as before. The second uses %xmm6-%xmm8,%xmm11-%xmm13. (Add 6 to %xmm2 and +// below. Add 8 to %xmm3 and up.) Instructions in the second instance are +// indented by one space. +// +// .p2align 4 .alt_entry _vpaes_encrypt_core_2x @@ -147,19 +147,19 @@ movl 240(%rdx),%eax movdqa %xmm9,%xmm1 movdqa %xmm9,%xmm7 - movdqa L$k_ipt(%rip),%xmm2 + movdqa L$k_ipt(%rip),%xmm2 // iptlo movdqa %xmm2,%xmm8 pandn %xmm0,%xmm1 pandn %xmm6,%xmm7 - movdqu (%r9),%xmm5 - + movdqu (%r9),%xmm5 // round0 key +// Also use %xmm5 in the second instance. psrld $4,%xmm1 psrld $4,%xmm7 pand %xmm9,%xmm0 pand %xmm9,%xmm6 pshufb %xmm0,%xmm2 pshufb %xmm6,%xmm8 - movdqa L$k_ipt+16(%rip),%xmm0 + movdqa L$k_ipt+16(%rip),%xmm0 // ipthi movdqa %xmm0,%xmm6 pshufb %xmm1,%xmm0 pshufb %xmm7,%xmm6 @@ -173,110 +173,110 @@ .p2align 4 L$enc2x_loop: - - movdqa L$k_sb1(%rip),%xmm4 - movdqa L$k_sb1+16(%rip),%xmm0 +// middle of middle round + movdqa L$k_sb1(%rip),%xmm4 // 4 : sb1u + movdqa L$k_sb1+16(%rip),%xmm0 // 0 : sb1t movdqa %xmm4,%xmm12 movdqa %xmm0,%xmm6 - pshufb %xmm2,%xmm4 + pshufb %xmm2,%xmm4 // 4 = sb1u pshufb %xmm8,%xmm12 - pshufb %xmm3,%xmm0 + pshufb %xmm3,%xmm0 // 0 = sb1t pshufb %xmm11,%xmm6 - pxor %xmm5,%xmm4 + pxor %xmm5,%xmm4 // 4 = sb1u + k pxor %xmm5,%xmm12 - movdqa L$k_sb2(%rip),%xmm5 + movdqa L$k_sb2(%rip),%xmm5 // 4 : sb2u movdqa %xmm5,%xmm13 - pxor %xmm4,%xmm0 + pxor %xmm4,%xmm0 // 0 = A pxor %xmm12,%xmm6 - movdqa -64(%r11,%r10,1),%xmm1 - - pshufb %xmm2,%xmm5 + movdqa -64(%r11,%r10,1),%xmm1 // .Lk_mc_forward[] +// Also use %xmm1 in the second instance. + pshufb %xmm2,%xmm5 // 4 = sb2u pshufb %xmm8,%xmm13 - movdqa (%r11,%r10,1),%xmm4 - - movdqa L$k_sb2+16(%rip),%xmm2 + movdqa (%r11,%r10,1),%xmm4 // .Lk_mc_backward[] +// Also use %xmm4 in the second instance. + movdqa L$k_sb2+16(%rip),%xmm2 // 2 : sb2t movdqa %xmm2,%xmm8 - pshufb %xmm3,%xmm2 + pshufb %xmm3,%xmm2 // 2 = sb2t pshufb %xmm11,%xmm8 - movdqa %xmm0,%xmm3 + movdqa %xmm0,%xmm3 // 3 = A movdqa %xmm6,%xmm11 - pxor %xmm5,%xmm2 + pxor %xmm5,%xmm2 // 2 = 2A pxor %xmm13,%xmm8 - pshufb %xmm1,%xmm0 + pshufb %xmm1,%xmm0 // 0 = B pshufb %xmm1,%xmm6 - addq $16,%r9 - pxor %xmm2,%xmm0 + addq $16,%r9 // next key + pxor %xmm2,%xmm0 // 0 = 2A+B pxor %xmm8,%xmm6 - pshufb %xmm4,%xmm3 + pshufb %xmm4,%xmm3 // 3 = D pshufb %xmm4,%xmm11 - addq $16,%r11 - pxor %xmm0,%xmm3 + addq $16,%r11 // next mc + pxor %xmm0,%xmm3 // 3 = 2A+B+D pxor %xmm6,%xmm11 - pshufb %xmm1,%xmm0 + pshufb %xmm1,%xmm0 // 0 = 2B+C pshufb %xmm1,%xmm6 - andq $0x30,%r11 - subq $1,%rax - pxor %xmm3,%xmm0 + andq $0x30,%r11 // ... mod 4 + subq $1,%rax // nr-- + pxor %xmm3,%xmm0 // 0 = 2A+3B+C+D pxor %xmm11,%xmm6 L$enc2x_entry: - - movdqa %xmm9,%xmm1 +// top of round + movdqa %xmm9,%xmm1 // 1 : i movdqa %xmm9,%xmm7 - movdqa L$k_inv+16(%rip),%xmm5 + movdqa L$k_inv+16(%rip),%xmm5 // 2 : a/k movdqa %xmm5,%xmm13 - pandn %xmm0,%xmm1 + pandn %xmm0,%xmm1 // 1 = i<<4 pandn %xmm6,%xmm7 - psrld $4,%xmm1 + psrld $4,%xmm1 // 1 = i psrld $4,%xmm7 - pand %xmm9,%xmm0 + pand %xmm9,%xmm0 // 0 = k pand %xmm9,%xmm6 - pshufb %xmm0,%xmm5 + pshufb %xmm0,%xmm5 // 2 = a/k pshufb %xmm6,%xmm13 - movdqa %xmm10,%xmm3 + movdqa %xmm10,%xmm3 // 3 : 1/i movdqa %xmm10,%xmm11 - pxor %xmm1,%xmm0 + pxor %xmm1,%xmm0 // 0 = j pxor %xmm7,%xmm6 - pshufb %xmm1,%xmm3 + pshufb %xmm1,%xmm3 // 3 = 1/i pshufb %xmm7,%xmm11 - movdqa %xmm10,%xmm4 + movdqa %xmm10,%xmm4 // 4 : 1/j movdqa %xmm10,%xmm12 - pxor %xmm5,%xmm3 + pxor %xmm5,%xmm3 // 3 = iak = 1/i + a/k pxor %xmm13,%xmm11 - pshufb %xmm0,%xmm4 + pshufb %xmm0,%xmm4 // 4 = 1/j pshufb %xmm6,%xmm12 - movdqa %xmm10,%xmm2 + movdqa %xmm10,%xmm2 // 2 : 1/iak movdqa %xmm10,%xmm8 - pxor %xmm5,%xmm4 + pxor %xmm5,%xmm4 // 4 = jak = 1/j + a/k pxor %xmm13,%xmm12 - pshufb %xmm3,%xmm2 + pshufb %xmm3,%xmm2 // 2 = 1/iak pshufb %xmm11,%xmm8 - movdqa %xmm10,%xmm3 + movdqa %xmm10,%xmm3 // 3 : 1/jak movdqa %xmm10,%xmm11 - pxor %xmm0,%xmm2 + pxor %xmm0,%xmm2 // 2 = io pxor %xmm6,%xmm8 - pshufb %xmm4,%xmm3 + pshufb %xmm4,%xmm3 // 3 = 1/jak pshufb %xmm12,%xmm11 movdqu (%r9),%xmm5 - - pxor %xmm1,%xmm3 +// Also use %xmm5 in the second instance. + pxor %xmm1,%xmm3 // 3 = jo pxor %xmm7,%xmm11 jnz L$enc2x_loop - - movdqa -96(%r10),%xmm4 - movdqa -80(%r10),%xmm0 +// middle of last round + movdqa -96(%r10),%xmm4 // 3 : sbou .Lk_sbo + movdqa -80(%r10),%xmm0 // 0 : sbot .Lk_sbo+16 movdqa %xmm4,%xmm12 movdqa %xmm0,%xmm6 - pshufb %xmm2,%xmm4 + pshufb %xmm2,%xmm4 // 4 = sbou pshufb %xmm8,%xmm12 - pxor %xmm5,%xmm4 + pxor %xmm5,%xmm4 // 4 = sb1u + k pxor %xmm5,%xmm12 - pshufb %xmm3,%xmm0 + pshufb %xmm3,%xmm0 // 0 = sb1t pshufb %xmm11,%xmm6 - movdqa 64(%r11,%r10,1),%xmm1 - - pxor %xmm4,%xmm0 + movdqa 64(%r11,%r10,1),%xmm1 // .Lk_sr[] +// Also use %xmm1 in the second instance. + pxor %xmm4,%xmm0 // 0 = A pxor %xmm12,%xmm6 pshufb %xmm1,%xmm0 pshufb %xmm1,%xmm6 @@ -284,28 +284,28 @@ - - - - - +// +// Decryption core +// +// Same API as encryption core. +// .p2align 4 .alt_entry _vpaes_decrypt_core _vpaes_decrypt_core: - movq %rdx,%r9 + movq %rdx,%r9 // load key movl 240(%rdx),%eax movdqa %xmm9,%xmm1 - movdqa L$k_dipt(%rip),%xmm2 + movdqa L$k_dipt(%rip),%xmm2 // iptlo pandn %xmm0,%xmm1 movq %rax,%r11 psrld $4,%xmm1 - movdqu (%r9),%xmm5 + movdqu (%r9),%xmm5 // round0 key shlq $4,%r11 pand %xmm9,%xmm0 pshufb %xmm0,%xmm2 - movdqa L$k_dipt+16(%rip),%xmm0 + movdqa L$k_dipt+16(%rip),%xmm0 // ipthi xorq $0x30,%r11 leaq L$k_dsbd(%rip),%r10 pshufb %xmm1,%xmm0 @@ -319,100 +319,100 @@ .p2align 4 L$dec_loop: - - - - movdqa -32(%r10),%xmm4 - movdqa -16(%r10),%xmm1 - pshufb %xmm2,%xmm4 - pshufb %xmm3,%xmm1 +// +// Inverse mix columns +// + movdqa -32(%r10),%xmm4 // 4 : sb9u + movdqa -16(%r10),%xmm1 // 0 : sb9t + pshufb %xmm2,%xmm4 // 4 = sb9u + pshufb %xmm3,%xmm1 // 0 = sb9t pxor %xmm4,%xmm0 - movdqa 0(%r10),%xmm4 - pxor %xmm1,%xmm0 - movdqa 16(%r10),%xmm1 + movdqa 0(%r10),%xmm4 // 4 : sbdu + pxor %xmm1,%xmm0 // 0 = ch + movdqa 16(%r10),%xmm1 // 0 : sbdt - pshufb %xmm2,%xmm4 - pshufb %xmm5,%xmm0 - pshufb %xmm3,%xmm1 - pxor %xmm4,%xmm0 - movdqa 32(%r10),%xmm4 - pxor %xmm1,%xmm0 - movdqa 48(%r10),%xmm1 + pshufb %xmm2,%xmm4 // 4 = sbdu + pshufb %xmm5,%xmm0 // MC ch + pshufb %xmm3,%xmm1 // 0 = sbdt + pxor %xmm4,%xmm0 // 4 = ch + movdqa 32(%r10),%xmm4 // 4 : sbbu + pxor %xmm1,%xmm0 // 0 = ch + movdqa 48(%r10),%xmm1 // 0 : sbbt - pshufb %xmm2,%xmm4 - pshufb %xmm5,%xmm0 - pshufb %xmm3,%xmm1 - pxor %xmm4,%xmm0 - movdqa 64(%r10),%xmm4 - pxor %xmm1,%xmm0 - movdqa 80(%r10),%xmm1 + pshufb %xmm2,%xmm4 // 4 = sbbu + pshufb %xmm5,%xmm0 // MC ch + pshufb %xmm3,%xmm1 // 0 = sbbt + pxor %xmm4,%xmm0 // 4 = ch + movdqa 64(%r10),%xmm4 // 4 : sbeu + pxor %xmm1,%xmm0 // 0 = ch + movdqa 80(%r10),%xmm1 // 0 : sbet - pshufb %xmm2,%xmm4 - pshufb %xmm5,%xmm0 - pshufb %xmm3,%xmm1 - pxor %xmm4,%xmm0 - addq $16,%r9 + pshufb %xmm2,%xmm4 // 4 = sbeu + pshufb %xmm5,%xmm0 // MC ch + pshufb %xmm3,%xmm1 // 0 = sbet + pxor %xmm4,%xmm0 // 4 = ch + addq $16,%r9 // next round key palignr $12,%xmm5,%xmm5 - pxor %xmm1,%xmm0 - subq $1,%rax + pxor %xmm1,%xmm0 // 0 = ch + subq $1,%rax // nr-- L$dec_entry: - - movdqa %xmm9,%xmm1 - pandn %xmm0,%xmm1 - movdqa %xmm11,%xmm2 - psrld $4,%xmm1 - pand %xmm9,%xmm0 - pshufb %xmm0,%xmm2 - movdqa %xmm10,%xmm3 - pxor %xmm1,%xmm0 - pshufb %xmm1,%xmm3 - movdqa %xmm10,%xmm4 - pxor %xmm2,%xmm3 - pshufb %xmm0,%xmm4 - pxor %xmm2,%xmm4 - movdqa %xmm10,%xmm2 - pshufb %xmm3,%xmm2 - movdqa %xmm10,%xmm3 - pxor %xmm0,%xmm2 - pshufb %xmm4,%xmm3 +// top of round + movdqa %xmm9,%xmm1 // 1 : i + pandn %xmm0,%xmm1 // 1 = i<<4 + movdqa %xmm11,%xmm2 // 2 : a/k + psrld $4,%xmm1 // 1 = i + pand %xmm9,%xmm0 // 0 = k + pshufb %xmm0,%xmm2 // 2 = a/k + movdqa %xmm10,%xmm3 // 3 : 1/i + pxor %xmm1,%xmm0 // 0 = j + pshufb %xmm1,%xmm3 // 3 = 1/i + movdqa %xmm10,%xmm4 // 4 : 1/j + pxor %xmm2,%xmm3 // 3 = iak = 1/i + a/k + pshufb %xmm0,%xmm4 // 4 = 1/j + pxor %xmm2,%xmm4 // 4 = jak = 1/j + a/k + movdqa %xmm10,%xmm2 // 2 : 1/iak + pshufb %xmm3,%xmm2 // 2 = 1/iak + movdqa %xmm10,%xmm3 // 3 : 1/jak + pxor %xmm0,%xmm2 // 2 = io + pshufb %xmm4,%xmm3 // 3 = 1/jak movdqu (%r9),%xmm0 - pxor %xmm1,%xmm3 + pxor %xmm1,%xmm3 // 3 = jo jnz L$dec_loop - - movdqa 96(%r10),%xmm4 - pshufb %xmm2,%xmm4 - pxor %xmm0,%xmm4 - movdqa 112(%r10),%xmm0 - movdqa -352(%r11),%xmm2 - pshufb %xmm3,%xmm0 - pxor %xmm4,%xmm0 +// middle of last round + movdqa 96(%r10),%xmm4 // 3 : sbou + pshufb %xmm2,%xmm4 // 4 = sbou + pxor %xmm0,%xmm4 // 4 = sb1u + k + movdqa 112(%r10),%xmm0 // 0 : sbot + movdqa -352(%r11),%xmm2 // .Lk_sr-.Lk_dsbd=-0x160 + pshufb %xmm3,%xmm0 // 0 = sb1t + pxor %xmm4,%xmm0 // 0 = A pshufb %xmm2,%xmm0 ret - - - - - +// ###################################################### +// ## +// AES key schedule ## +// ## +// ###################################################### .p2align 4 .alt_entry _vpaes_schedule_core _vpaes_schedule_core: +// rdi = key +// rsi = size in bits +// rdx = buffer +// rcx = direction. 0=encrypt, 1=decrypt + call _vpaes_preheat // load the tables + movdqa L$k_rcon(%rip),%xmm8 // load rcon + movdqu (%rdi),%xmm0 // load key (unaligned) - - - - call _vpaes_preheat - movdqa L$k_rcon(%rip),%xmm8 - movdqu (%rdi),%xmm0 - - +// input transform movdqa %xmm0,%xmm3 leaq L$k_ipt(%rip),%r11 call _vpaes_schedule_transform @@ -422,12 +422,12 @@ testq %rcx,%rcx jnz L$schedule_am_decrypting - +// encrypting, output zeroth round key after transform movdqu %xmm0,(%rdx) jmp L$schedule_go L$schedule_am_decrypting: - +// decrypting, output zeroth round key after shiftrows movdqa (%r8,%r10,1),%xmm1 pshufb %xmm1,%xmm3 movdqu %xmm3,(%rdx) @@ -437,16 +437,16 @@ cmpl $192,%esi ja L$schedule_256 je L$schedule_192 +// 128: fall though - - - - - - - - - +// +// .schedule_128 +// +// 128-bit specific part of key schedule. +// +// This schedule is really simple, because all its parts +// are accomplished by the subroutines. +// L$schedule_128: movl $10,%esi @@ -454,73 +454,73 @@ call _vpaes_schedule_round decq %rsi jz L$schedule_mangle_last - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // write output jmp L$oop_schedule_128 - - - - - - - - - - - - - - - +// +// .aes_schedule_192 +// +// 192-bit specific part of key schedule. +// +// The main body of this schedule is the same as the 128-bit +// schedule, but with more smearing. The long, high side is +// stored in %xmm7 as before, and the short, low side is in +// the high bits of %xmm6. +// +// This schedule is somewhat nastier, however, because each +// round produces 192 bits of key material, or 1.5 round keys. +// Therefore, on each cycle we do 2 rounds and produce 3 round +// keys. +// .p2align 4 L$schedule_192: - movdqu 8(%rdi),%xmm0 - call _vpaes_schedule_transform - movdqa %xmm0,%xmm6 - pxor %xmm4,%xmm4 - movhlps %xmm4,%xmm6 + movdqu 8(%rdi),%xmm0 // load key part 2 (very unaligned) + call _vpaes_schedule_transform // input transform + movdqa %xmm0,%xmm6 // save short part + pxor %xmm4,%xmm4 // clear 4 + movhlps %xmm4,%xmm6 // clobber low side with zeros movl $4,%esi L$oop_schedule_192: call _vpaes_schedule_round palignr $8,%xmm6,%xmm0 - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // save key n call _vpaes_schedule_192_smear - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // save key n+1 call _vpaes_schedule_round decq %rsi jz L$schedule_mangle_last - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // save key n+2 call _vpaes_schedule_192_smear jmp L$oop_schedule_192 - - - - - - - - - - +// +// .aes_schedule_256 +// +// 256-bit specific part of key schedule. +// +// The structure here is very similar to the 128-bit +// schedule, but with an additional "low side" in +// %xmm6. The low side's rounds are the same as the +// high side's, except no rcon and no rotation. +// .p2align 4 L$schedule_256: - movdqu 16(%rdi),%xmm0 - call _vpaes_schedule_transform + movdqu 16(%rdi),%xmm0 // load key part 2 (unaligned) + call _vpaes_schedule_transform // input transform movl $7,%esi L$oop_schedule_256: - call _vpaes_schedule_mangle - movdqa %xmm0,%xmm6 + call _vpaes_schedule_mangle // output low result + movdqa %xmm0,%xmm6 // save cur_lo in xmm6 - +// high round call _vpaes_schedule_round decq %rsi jz L$schedule_mangle_last call _vpaes_schedule_mangle - +// low round. swap xmm7 and xmm6 pshufd $0xFF,%xmm0,%xmm0 movdqa %xmm7,%xmm5 movdqa %xmm6,%xmm7 @@ -530,36 +530,36 @@ jmp L$oop_schedule_256 - - - - - - - - - - +// +// .aes_schedule_mangle_last +// +// Mangler for last round of key schedule +// Mangles %xmm0 +// when encrypting, outputs out(%xmm0) ^ 63 +// when decrypting, outputs unskew(%xmm0) +// +// Always called right before return... jumps to cleanup and exits +// .p2align 4 L$schedule_mangle_last: - - leaq L$k_deskew(%rip),%r11 +// schedule last round key from xmm0 + leaq L$k_deskew(%rip),%r11 // prepare to deskew testq %rcx,%rcx jnz L$schedule_mangle_last_dec - +// encrypting movdqa (%r8,%r10,1),%xmm1 - pshufb %xmm1,%xmm0 - leaq L$k_opt(%rip),%r11 + pshufb %xmm1,%xmm0 // output permute + leaq L$k_opt(%rip),%r11 // prepare to output transform addq $32,%rdx L$schedule_mangle_last_dec: addq $-16,%rdx pxor L$k_s63(%rip),%xmm0 - call _vpaes_schedule_transform - movdqu %xmm0,(%rdx) + call _vpaes_schedule_transform // output transform + movdqu %xmm0,(%rdx) // save last key - +// cleanup pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -572,75 +572,75 @@ - - - - - - - - - - - - - - +// +// .aes_schedule_192_smear +// +// Smear the short, low side in the 192-bit key schedule. +// +// Inputs: +// %xmm7: high side, b a x y +// %xmm6: low side, d c 0 0 +// %xmm13: 0 +// +// Outputs: +// %xmm6: b+c+d b+c 0 0 +// %xmm0: b+c+d b+c b a +// .p2align 4 .alt_entry _vpaes_schedule_192_smear _vpaes_schedule_192_smear: - pshufd $0x80,%xmm6,%xmm1 - pshufd $0xFE,%xmm7,%xmm0 - pxor %xmm1,%xmm6 + pshufd $0x80,%xmm6,%xmm1 // d c 0 0 -> c 0 0 0 + pshufd $0xFE,%xmm7,%xmm0 // b a _ _ -> b b b a + pxor %xmm1,%xmm6 // -> c+d c 0 0 pxor %xmm1,%xmm1 - pxor %xmm0,%xmm6 + pxor %xmm0,%xmm6 // -> b+c+d b+c b a movdqa %xmm6,%xmm0 - movhlps %xmm1,%xmm6 + movhlps %xmm1,%xmm6 // clobber low side with zeros ret - - - - - - - - - - - - - - - - - - +// +// .aes_schedule_round +// +// Runs one main round of the key schedule on %xmm0, %xmm7 +// +// Specifically, runs subbytes on the high dword of %xmm0 +// then rotates it by one byte and xors into the low dword of +// %xmm7. +// +// Adds rcon from low byte of %xmm8, then rotates %xmm8 for +// next rcon. +// +// Smears the dwords of %xmm7 by xoring the low into the +// second low, result into third, result into highest. +// +// Returns results in %xmm7 = %xmm0. +// Clobbers %xmm1-%xmm4, %r11. +// .p2align 4 .alt_entry _vpaes_schedule_round _vpaes_schedule_round: - +// extract rcon from xmm8 pxor %xmm1,%xmm1 palignr $15,%xmm8,%xmm1 palignr $15,%xmm8,%xmm8 pxor %xmm1,%xmm7 - +// rotate pshufd $0xFF,%xmm0,%xmm0 palignr $1,%xmm0,%xmm0 +// fall through... - - +// low round: same as high round, but no rotation and no rcon. .alt_entry _vpaes_schedule_low_round _vpaes_schedule_low_round: - +// smear xmm7 movdqa %xmm7,%xmm1 pslldq $4,%xmm7 pxor %xmm1,%xmm7 @@ -649,48 +649,48 @@ pxor %xmm1,%xmm7 pxor L$k_s63(%rip),%xmm7 - +// subbytes movdqa %xmm9,%xmm1 pandn %xmm0,%xmm1 - psrld $4,%xmm1 - pand %xmm9,%xmm0 - movdqa %xmm11,%xmm2 - pshufb %xmm0,%xmm2 - pxor %xmm1,%xmm0 - movdqa %xmm10,%xmm3 - pshufb %xmm1,%xmm3 - pxor %xmm2,%xmm3 - movdqa %xmm10,%xmm4 - pshufb %xmm0,%xmm4 - pxor %xmm2,%xmm4 - movdqa %xmm10,%xmm2 - pshufb %xmm3,%xmm2 - pxor %xmm0,%xmm2 - movdqa %xmm10,%xmm3 - pshufb %xmm4,%xmm3 - pxor %xmm1,%xmm3 - movdqa %xmm13,%xmm4 - pshufb %xmm2,%xmm4 - movdqa %xmm12,%xmm0 - pshufb %xmm3,%xmm0 - pxor %xmm4,%xmm0 + psrld $4,%xmm1 // 1 = i + pand %xmm9,%xmm0 // 0 = k + movdqa %xmm11,%xmm2 // 2 : a/k + pshufb %xmm0,%xmm2 // 2 = a/k + pxor %xmm1,%xmm0 // 0 = j + movdqa %xmm10,%xmm3 // 3 : 1/i + pshufb %xmm1,%xmm3 // 3 = 1/i + pxor %xmm2,%xmm3 // 3 = iak = 1/i + a/k + movdqa %xmm10,%xmm4 // 4 : 1/j + pshufb %xmm0,%xmm4 // 4 = 1/j + pxor %xmm2,%xmm4 // 4 = jak = 1/j + a/k + movdqa %xmm10,%xmm2 // 2 : 1/iak + pshufb %xmm3,%xmm2 // 2 = 1/iak + pxor %xmm0,%xmm2 // 2 = io + movdqa %xmm10,%xmm3 // 3 : 1/jak + pshufb %xmm4,%xmm3 // 3 = 1/jak + pxor %xmm1,%xmm3 // 3 = jo + movdqa %xmm13,%xmm4 // 4 : sbou + pshufb %xmm2,%xmm4 // 4 = sbou + movdqa %xmm12,%xmm0 // 0 : sbot + pshufb %xmm3,%xmm0 // 0 = sb1t + pxor %xmm4,%xmm0 // 0 = sbox output - +// add in smeared stuff pxor %xmm7,%xmm0 movdqa %xmm0,%xmm7 ret - - - - - - - - - +// +// .aes_schedule_transform +// +// Linear-transform %xmm0 according to tables at (%r11) +// +// Requires that %xmm9 = 0x0F0F... as in preheat +// Output in %xmm0 +// Clobbers %xmm1, %xmm2 +// .p2align 4 .alt_entry _vpaes_schedule_transform @@ -700,49 +700,49 @@ pandn %xmm0,%xmm1 psrld $4,%xmm1 pand %xmm9,%xmm0 - movdqa (%r11),%xmm2 + movdqa (%r11),%xmm2 // lo pshufb %xmm0,%xmm2 - movdqa 16(%r11),%xmm0 + movdqa 16(%r11),%xmm0 // hi pshufb %xmm1,%xmm0 pxor %xmm2,%xmm0 ret - - - - - - - - - - - - - - - - - - - - - - - +// +// .aes_schedule_mangle +// +// Mangle xmm0 from (basis-transformed) standard version +// to our version. +// +// On encrypt, +// xor with 0x63 +// multiply by circulant 0,1,1,1 +// apply shiftrows transform +// +// On decrypt, +// xor with 0x63 +// multiply by "inverse mixcolumns" circulant E,B,D,9 +// deskew +// apply shiftrows transform +// +// +// Writes out to (%rdx), and increments or decrements it +// Keeps track of round number mod 4 in %r8 +// Preserves xmm0 +// Clobbers xmm1-xmm5 +// .p2align 4 .alt_entry _vpaes_schedule_mangle _vpaes_schedule_mangle: - movdqa %xmm0,%xmm4 + movdqa %xmm0,%xmm4 // save xmm0 for later movdqa L$k_mc_forward(%rip),%xmm5 testq %rcx,%rcx jnz L$schedule_mangle_dec - +// encrypting addq $16,%rdx pxor L$k_s63(%rip),%xmm4 pshufb %xmm5,%xmm4 @@ -755,12 +755,12 @@ jmp L$schedule_mangle_both .p2align 4 L$schedule_mangle_dec: - +// inverse mix columns leaq L$k_dksd(%rip),%r11 movdqa %xmm9,%xmm1 pandn %xmm4,%xmm1 - psrld $4,%xmm1 - pand %xmm9,%xmm4 + psrld $4,%xmm1 // 1 = hi + pand %xmm9,%xmm4 // 4 = lo movdqa 0(%r11),%xmm2 pshufb %xmm4,%xmm2 @@ -804,9 +804,9 @@ - - - +// +// Interface to OpenSSL +// .globl _vpaes_set_encrypt_key .private_extern _vpaes_set_encrypt_key @@ -823,7 +823,7 @@ movl %esi,%eax shrl $5,%eax addl $5,%eax - movl %eax,240(%rdx) + movl %eax,240(%rdx) // AES_KEY->rounds = nbits/32+5; movl $0,%ecx movl $0x30,%r8d @@ -844,7 +844,7 @@ movl %esi,%eax shrl $5,%eax addl $5,%eax - movl %eax,240(%rdx) + movl %eax,240(%rdx) // AES_KEY->rounds = nbits/32+5; shll $4,%eax leaq 16(%rdx,%rax,1),%rdx @@ -852,7 +852,7 @@ movl %esi,%r8d shrl $1,%r8d andl $32,%r8d - xorl $32,%r8d + xorl $32,%r8d // nbits==192?0:32 call _vpaes_schedule_core xorl %eax,%eax ret @@ -905,7 +905,7 @@ xchgq %rcx,%rdx subq $16,%rcx jc L$cbc_abort - movdqu (%r8),%xmm6 + movdqu (%r8),%xmm6 // load IV subq %rdi,%rsi call _vpaes_preheat cmpl $0,%r9d @@ -934,7 +934,7 @@ subq $16,%rcx jnc L$cbc_dec_loop L$cbc_done: - movdqu %xmm6,(%r8) + movdqu %xmm6,(%r8) // save IV L$cbc_abort: ret @@ -947,13 +947,13 @@ _vpaes_ctr32_encrypt_blocks: _CET_ENDBR - +// _vpaes_encrypt_core and _vpaes_encrypt_core_2x expect the key in %rdx. xchgq %rcx,%rdx testq %rcx,%rcx jz L$ctr32_abort - movdqu (%r8),%xmm0 + movdqu (%r8),%xmm0 // Load IV. movdqa L$ctr_add_one(%rip),%xmm8 - subq %rdi,%rsi + subq %rdi,%rsi // This allows only incrementing %rdi. call _vpaes_preheat movdqa %xmm0,%xmm6 pshufb L$rev_ctr(%rip),%xmm6 @@ -961,9 +961,9 @@ testq $1,%rcx jz L$ctr32_prep_loop - - - movdqu (%rdi),%xmm7 +// Handle one block so the remaining block count is even for +// _vpaes_encrypt_core_2x. + movdqu (%rdi),%xmm7 // Load input. call _vpaes_encrypt_core pxor %xmm7,%xmm0 paddd %xmm8,%xmm6 @@ -973,29 +973,29 @@ jz L$ctr32_done L$ctr32_prep_loop: - - +// _vpaes_encrypt_core_2x leaves only %xmm14 and %xmm15 as spare +// registers. We maintain two byte-swapped counters in them. movdqa %xmm6,%xmm14 movdqa %xmm6,%xmm15 paddd %xmm8,%xmm15 L$ctr32_loop: - movdqa L$rev_ctr(%rip),%xmm1 + movdqa L$rev_ctr(%rip),%xmm1 // Set up counters. movdqa %xmm14,%xmm0 movdqa %xmm15,%xmm6 pshufb %xmm1,%xmm0 pshufb %xmm1,%xmm6 call _vpaes_encrypt_core_2x - movdqu (%rdi),%xmm1 + movdqu (%rdi),%xmm1 // Load input. movdqu 16(%rdi),%xmm2 movdqa L$ctr_add_two(%rip),%xmm3 - pxor %xmm1,%xmm0 + pxor %xmm1,%xmm0 // XOR input. pxor %xmm2,%xmm6 - paddd %xmm3,%xmm14 + paddd %xmm3,%xmm14 // Increment counters. paddd %xmm3,%xmm15 - movdqu %xmm0,(%rsi,%rdi,1) + movdqu %xmm0,(%rsi,%rdi,1) // Write output. movdqu %xmm6,16(%rsi,%rdi,1) - subq $2,%rcx + subq $2,%rcx // Advance loop. leaq 32(%rdi),%rdi jnz L$ctr32_loop @@ -1004,136 +1004,136 @@ ret - - - - - - +// +// _aes_preheat +// +// Fills register %r10 -> .aes_consts (so you can -fPIC) +// and %xmm9-%xmm15 as specified below. +// .p2align 4 .alt_entry _vpaes_preheat _vpaes_preheat: leaq L$k_s0F(%rip),%r10 - movdqa -32(%r10),%xmm10 - movdqa -16(%r10),%xmm11 - movdqa 0(%r10),%xmm9 - movdqa 48(%r10),%xmm13 - movdqa 64(%r10),%xmm12 - movdqa 80(%r10),%xmm15 - movdqa 96(%r10),%xmm14 + movdqa -32(%r10),%xmm10 // .Lk_inv + movdqa -16(%r10),%xmm11 // .Lk_inv+16 + movdqa 0(%r10),%xmm9 // .Lk_s0F + movdqa 48(%r10),%xmm13 // .Lk_sb1 + movdqa 64(%r10),%xmm12 // .Lk_sb1+16 + movdqa 80(%r10),%xmm15 // .Lk_sb2 + movdqa 96(%r10),%xmm14 // .Lk_sb2+16 ret - - - - - +// ###################################################### +// ## +// Constants ## +// ## +// ###################################################### .section __DATA,__const .p2align 6 _vpaes_consts: -L$k_inv: +L$k_inv: // inv, inva .quad 0x0E05060F0D080180, 0x040703090A0B0C02 .quad 0x01040A060F0B0780, 0x030D0E0C02050809 -L$k_s0F: +L$k_s0F: // s0F .quad 0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F0F0F0F0F -L$k_ipt: +L$k_ipt: // input transform (lo, hi) .quad 0xC2B2E8985A2A7000, 0xCABAE09052227808 .quad 0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81 -L$k_sb1: +L$k_sb1: // sb1u, sb1t .quad 0xB19BE18FCB503E00, 0xA5DF7A6E142AF544 .quad 0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF -L$k_sb2: +L$k_sb2: // sb2u, sb2t .quad 0xE27A93C60B712400, 0x5EB7E955BC982FCD .quad 0x69EB88400AE12900, 0xC2A163C8AB82234A -L$k_sbo: +L$k_sbo: // sbou, sbot .quad 0xD0D26D176FBDC700, 0x15AABF7AC502A878 .quad 0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA -L$k_mc_forward: +L$k_mc_forward: // mc_forward .quad 0x0407060500030201, 0x0C0F0E0D080B0A09 .quad 0x080B0A0904070605, 0x000302010C0F0E0D .quad 0x0C0F0E0D080B0A09, 0x0407060500030201 .quad 0x000302010C0F0E0D, 0x080B0A0904070605 -L$k_mc_backward: +L$k_mc_backward: // mc_backward .quad 0x0605040702010003, 0x0E0D0C0F0A09080B .quad 0x020100030E0D0C0F, 0x0A09080B06050407 .quad 0x0E0D0C0F0A09080B, 0x0605040702010003 .quad 0x0A09080B06050407, 0x020100030E0D0C0F -L$k_sr: +L$k_sr: // sr .quad 0x0706050403020100, 0x0F0E0D0C0B0A0908 .quad 0x030E09040F0A0500, 0x0B06010C07020D08 .quad 0x0F060D040B020900, 0x070E050C030A0108 .quad 0x0B0E0104070A0D00, 0x0306090C0F020508 -L$k_rcon: +L$k_rcon: // rcon .quad 0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81 -L$k_s63: +L$k_s63: // s63: all equal to 0x63 transformed .quad 0x5B5B5B5B5B5B5B5B, 0x5B5B5B5B5B5B5B5B -L$k_opt: +L$k_opt: // output transform .quad 0xFF9F4929D6B66000, 0xF7974121DEBE6808 .quad 0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0 -L$k_deskew: +L$k_deskew: // deskew tables: inverts the sbox's "skew" .quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A .quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77 - - - - -L$k_dksd: +// +// Decryption stuff +// Key schedule constants +// +L$k_dksd: // decryption key schedule: invskew x*D .quad 0xFEB91A5DA3E44700, 0x0740E3A45A1DBEF9 .quad 0x41C277F4B5368300, 0x5FDC69EAAB289D1E -L$k_dksb: +L$k_dksb: // decryption key schedule: invskew x*B .quad 0x9A4FCA1F8550D500, 0x03D653861CC94C99 .quad 0x115BEDA7B6FC4A00, 0xD993256F7E3482C8 -L$k_dkse: +L$k_dkse: // decryption key schedule: invskew x*E + 0x63 .quad 0xD5031CCA1FC9D600, 0x53859A4C994F5086 .quad 0xA23196054FDC7BE8, 0xCD5EF96A20B31487 -L$k_dks9: +L$k_dks9: // decryption key schedule: invskew x*9 .quad 0xB6116FC87ED9A700, 0x4AED933482255BFC .quad 0x4576516227143300, 0x8BB89FACE9DAFDCE - - - - -L$k_dipt: +// +// Decryption stuff +// Round function constants +// +L$k_dipt: // decryption input transform .quad 0x0F505B040B545F00, 0x154A411E114E451A .quad 0x86E383E660056500, 0x12771772F491F194 -L$k_dsb9: +L$k_dsb9: // decryption sbox output *9*u, *9*t .quad 0x851C03539A86D600, 0xCAD51F504F994CC9 .quad 0xC03B1789ECD74900, 0x725E2C9EB2FBA565 -L$k_dsbd: +L$k_dsbd: // decryption sbox output *D*u, *D*t .quad 0x7D57CCDFE6B1A200, 0xF56E9B13882A4439 .quad 0x3CE2FAF724C6CB00, 0x2931180D15DEEFD3 -L$k_dsbb: +L$k_dsbb: // decryption sbox output *B*u, *B*t .quad 0xD022649296B44200, 0x602646F6B0F2D404 .quad 0xC19498A6CD596700, 0xF3FF0C3E3255AA6B -L$k_dsbe: +L$k_dsbe: // decryption sbox output *E*u, *E*t .quad 0x46F2929626D4D000, 0x2242600464B4F6B0 .quad 0x0C55A6CDFFAAC100, 0x9467F36B98593E32 -L$k_dsbo: +L$k_dsbo: // decryption sbox final output .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C - +// .Lrev_ctr is a permutation which byte-swaps the counter portion of the IV. L$rev_ctr: .quad 0x0706050403020100, 0x0c0d0e0f0b0a0908 - - +// .Lctr_add_* may be added to a byte-swapped xmm register to increment the +// counter. The register must be byte-swapped again to form the actual input. L$ctr_add_one: .quad 0x0000000000000000, 0x0000000100000000 L$ctr_add_two:
diff --git a/gen/bcm/vpaes-x86_64-linux.S b/gen/bcm/vpaes-x86_64-linux.S index e788464..db1f9fc 100644 --- a/gen/bcm/vpaes-x86_64-linux.S +++ b/gen/bcm/vpaes-x86_64-linux.S
@@ -6,21 +6,21 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) .text - - - - - - - - - - - - - - - +// +// _aes_encrypt_core +// +// AES-encrypt %xmm0. +// +// Inputs: +// %xmm0 = input +// %xmm9-%xmm15 as in _vpaes_preheat +// (%rdx) = scheduled keys +// +// Output in %xmm0 +// Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax +// Preserves %xmm6 - %xmm8 so you get some local vectors +// +// .type _vpaes_encrypt_core,@function .align 16 _vpaes_encrypt_core: @@ -29,13 +29,13 @@ movq $16,%r11 movl 240(%rdx),%eax movdqa %xmm9,%xmm1 - movdqa .Lk_ipt(%rip),%xmm2 + movdqa .Lk_ipt(%rip),%xmm2 // iptlo pandn %xmm0,%xmm1 - movdqu (%r9),%xmm5 + movdqu (%r9),%xmm5 // round0 key psrld $4,%xmm1 pand %xmm9,%xmm0 pshufb %xmm0,%xmm2 - movdqa .Lk_ipt+16(%rip),%xmm0 + movdqa .Lk_ipt+16(%rip),%xmm0 // ipthi pshufb %xmm1,%xmm0 pxor %xmm5,%xmm2 addq $16,%r9 @@ -45,98 +45,98 @@ .align 16 .Lenc_loop: - - movdqa %xmm13,%xmm4 - movdqa %xmm12,%xmm0 - pshufb %xmm2,%xmm4 - pshufb %xmm3,%xmm0 - pxor %xmm5,%xmm4 - movdqa %xmm15,%xmm5 - pxor %xmm4,%xmm0 - movdqa -64(%r11,%r10,1),%xmm1 - pshufb %xmm2,%xmm5 - movdqa (%r11,%r10,1),%xmm4 - movdqa %xmm14,%xmm2 - pshufb %xmm3,%xmm2 - movdqa %xmm0,%xmm3 - pxor %xmm5,%xmm2 - pshufb %xmm1,%xmm0 - addq $16,%r9 - pxor %xmm2,%xmm0 - pshufb %xmm4,%xmm3 - addq $16,%r11 - pxor %xmm0,%xmm3 - pshufb %xmm1,%xmm0 - andq $0x30,%r11 - subq $1,%rax - pxor %xmm3,%xmm0 +// middle of middle round + movdqa %xmm13,%xmm4 // 4 : sb1u + movdqa %xmm12,%xmm0 // 0 : sb1t + pshufb %xmm2,%xmm4 // 4 = sb1u + pshufb %xmm3,%xmm0 // 0 = sb1t + pxor %xmm5,%xmm4 // 4 = sb1u + k + movdqa %xmm15,%xmm5 // 4 : sb2u + pxor %xmm4,%xmm0 // 0 = A + movdqa -64(%r11,%r10,1),%xmm1 // .Lk_mc_forward[] + pshufb %xmm2,%xmm5 // 4 = sb2u + movdqa (%r11,%r10,1),%xmm4 // .Lk_mc_backward[] + movdqa %xmm14,%xmm2 // 2 : sb2t + pshufb %xmm3,%xmm2 // 2 = sb2t + movdqa %xmm0,%xmm3 // 3 = A + pxor %xmm5,%xmm2 // 2 = 2A + pshufb %xmm1,%xmm0 // 0 = B + addq $16,%r9 // next key + pxor %xmm2,%xmm0 // 0 = 2A+B + pshufb %xmm4,%xmm3 // 3 = D + addq $16,%r11 // next mc + pxor %xmm0,%xmm3 // 3 = 2A+B+D + pshufb %xmm1,%xmm0 // 0 = 2B+C + andq $0x30,%r11 // ... mod 4 + subq $1,%rax // nr-- + pxor %xmm3,%xmm0 // 0 = 2A+3B+C+D .Lenc_entry: - - movdqa %xmm9,%xmm1 - movdqa %xmm11,%xmm5 - pandn %xmm0,%xmm1 - psrld $4,%xmm1 - pand %xmm9,%xmm0 - pshufb %xmm0,%xmm5 - movdqa %xmm10,%xmm3 - pxor %xmm1,%xmm0 - pshufb %xmm1,%xmm3 - movdqa %xmm10,%xmm4 - pxor %xmm5,%xmm3 - pshufb %xmm0,%xmm4 - movdqa %xmm10,%xmm2 - pxor %xmm5,%xmm4 - pshufb %xmm3,%xmm2 - movdqa %xmm10,%xmm3 - pxor %xmm0,%xmm2 - pshufb %xmm4,%xmm3 +// top of round + movdqa %xmm9,%xmm1 // 1 : i + movdqa %xmm11,%xmm5 // 2 : a/k + pandn %xmm0,%xmm1 // 1 = i<<4 + psrld $4,%xmm1 // 1 = i + pand %xmm9,%xmm0 // 0 = k + pshufb %xmm0,%xmm5 // 2 = a/k + movdqa %xmm10,%xmm3 // 3 : 1/i + pxor %xmm1,%xmm0 // 0 = j + pshufb %xmm1,%xmm3 // 3 = 1/i + movdqa %xmm10,%xmm4 // 4 : 1/j + pxor %xmm5,%xmm3 // 3 = iak = 1/i + a/k + pshufb %xmm0,%xmm4 // 4 = 1/j + movdqa %xmm10,%xmm2 // 2 : 1/iak + pxor %xmm5,%xmm4 // 4 = jak = 1/j + a/k + pshufb %xmm3,%xmm2 // 2 = 1/iak + movdqa %xmm10,%xmm3 // 3 : 1/jak + pxor %xmm0,%xmm2 // 2 = io + pshufb %xmm4,%xmm3 // 3 = 1/jak movdqu (%r9),%xmm5 - pxor %xmm1,%xmm3 + pxor %xmm1,%xmm3 // 3 = jo jnz .Lenc_loop - - movdqa -96(%r10),%xmm4 - movdqa -80(%r10),%xmm0 - pshufb %xmm2,%xmm4 - pxor %xmm5,%xmm4 - pshufb %xmm3,%xmm0 - movdqa 64(%r11,%r10,1),%xmm1 - pxor %xmm4,%xmm0 +// middle of last round + movdqa -96(%r10),%xmm4 // 3 : sbou .Lk_sbo + movdqa -80(%r10),%xmm0 // 0 : sbot .Lk_sbo+16 + pshufb %xmm2,%xmm4 // 4 = sbou + pxor %xmm5,%xmm4 // 4 = sb1u + k + pshufb %xmm3,%xmm0 // 0 = sb1t + movdqa 64(%r11,%r10,1),%xmm1 // .Lk_sr[] + pxor %xmm4,%xmm0 // 0 = A pshufb %xmm1,%xmm0 ret .cfi_endproc .size _vpaes_encrypt_core,.-_vpaes_encrypt_core - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// +// _aes_encrypt_core_2x +// +// AES-encrypt %xmm0 and %xmm6 in parallel. +// +// Inputs: +// %xmm0 and %xmm6 = input +// %xmm9 and %xmm10 as in _vpaes_preheat +// (%rdx) = scheduled keys +// +// Output in %xmm0 and %xmm6 +// Clobbers %xmm1-%xmm5, %xmm7, %xmm8, %xmm11-%xmm13, %r9, %r10, %r11, %rax +// Preserves %xmm14 and %xmm15 +// +// This function stitches two parallel instances of _vpaes_encrypt_core. x86_64 +// provides 16 XMM registers. _vpaes_encrypt_core computes over six registers +// (%xmm0-%xmm5) and additionally uses seven registers with preloaded constants +// from _vpaes_preheat (%xmm9-%xmm15). This does not quite fit two instances, +// so we spill some of %xmm9 through %xmm15 back to memory. We keep %xmm9 and +// %xmm10 in registers as these values are used several times in a row. The +// remainder are read once per round and are spilled to memory. This leaves two +// registers preserved for the caller. +// +// Thus, of the two _vpaes_encrypt_core instances, the first uses (%xmm0-%xmm5) +// as before. The second uses %xmm6-%xmm8,%xmm11-%xmm13. (Add 6 to %xmm2 and +// below. Add 8 to %xmm3 and up.) Instructions in the second instance are +// indented by one space. +// +// .type _vpaes_encrypt_core_2x,@function .align 16 _vpaes_encrypt_core_2x: @@ -146,19 +146,19 @@ movl 240(%rdx),%eax movdqa %xmm9,%xmm1 movdqa %xmm9,%xmm7 - movdqa .Lk_ipt(%rip),%xmm2 + movdqa .Lk_ipt(%rip),%xmm2 // iptlo movdqa %xmm2,%xmm8 pandn %xmm0,%xmm1 pandn %xmm6,%xmm7 - movdqu (%r9),%xmm5 - + movdqu (%r9),%xmm5 // round0 key +// Also use %xmm5 in the second instance. psrld $4,%xmm1 psrld $4,%xmm7 pand %xmm9,%xmm0 pand %xmm9,%xmm6 pshufb %xmm0,%xmm2 pshufb %xmm6,%xmm8 - movdqa .Lk_ipt+16(%rip),%xmm0 + movdqa .Lk_ipt+16(%rip),%xmm0 // ipthi movdqa %xmm0,%xmm6 pshufb %xmm1,%xmm0 pshufb %xmm7,%xmm6 @@ -172,110 +172,110 @@ .align 16 .Lenc2x_loop: - - movdqa .Lk_sb1(%rip),%xmm4 - movdqa .Lk_sb1+16(%rip),%xmm0 +// middle of middle round + movdqa .Lk_sb1(%rip),%xmm4 // 4 : sb1u + movdqa .Lk_sb1+16(%rip),%xmm0 // 0 : sb1t movdqa %xmm4,%xmm12 movdqa %xmm0,%xmm6 - pshufb %xmm2,%xmm4 + pshufb %xmm2,%xmm4 // 4 = sb1u pshufb %xmm8,%xmm12 - pshufb %xmm3,%xmm0 + pshufb %xmm3,%xmm0 // 0 = sb1t pshufb %xmm11,%xmm6 - pxor %xmm5,%xmm4 + pxor %xmm5,%xmm4 // 4 = sb1u + k pxor %xmm5,%xmm12 - movdqa .Lk_sb2(%rip),%xmm5 + movdqa .Lk_sb2(%rip),%xmm5 // 4 : sb2u movdqa %xmm5,%xmm13 - pxor %xmm4,%xmm0 + pxor %xmm4,%xmm0 // 0 = A pxor %xmm12,%xmm6 - movdqa -64(%r11,%r10,1),%xmm1 - - pshufb %xmm2,%xmm5 + movdqa -64(%r11,%r10,1),%xmm1 // .Lk_mc_forward[] +// Also use %xmm1 in the second instance. + pshufb %xmm2,%xmm5 // 4 = sb2u pshufb %xmm8,%xmm13 - movdqa (%r11,%r10,1),%xmm4 - - movdqa .Lk_sb2+16(%rip),%xmm2 + movdqa (%r11,%r10,1),%xmm4 // .Lk_mc_backward[] +// Also use %xmm4 in the second instance. + movdqa .Lk_sb2+16(%rip),%xmm2 // 2 : sb2t movdqa %xmm2,%xmm8 - pshufb %xmm3,%xmm2 + pshufb %xmm3,%xmm2 // 2 = sb2t pshufb %xmm11,%xmm8 - movdqa %xmm0,%xmm3 + movdqa %xmm0,%xmm3 // 3 = A movdqa %xmm6,%xmm11 - pxor %xmm5,%xmm2 + pxor %xmm5,%xmm2 // 2 = 2A pxor %xmm13,%xmm8 - pshufb %xmm1,%xmm0 + pshufb %xmm1,%xmm0 // 0 = B pshufb %xmm1,%xmm6 - addq $16,%r9 - pxor %xmm2,%xmm0 + addq $16,%r9 // next key + pxor %xmm2,%xmm0 // 0 = 2A+B pxor %xmm8,%xmm6 - pshufb %xmm4,%xmm3 + pshufb %xmm4,%xmm3 // 3 = D pshufb %xmm4,%xmm11 - addq $16,%r11 - pxor %xmm0,%xmm3 + addq $16,%r11 // next mc + pxor %xmm0,%xmm3 // 3 = 2A+B+D pxor %xmm6,%xmm11 - pshufb %xmm1,%xmm0 + pshufb %xmm1,%xmm0 // 0 = 2B+C pshufb %xmm1,%xmm6 - andq $0x30,%r11 - subq $1,%rax - pxor %xmm3,%xmm0 + andq $0x30,%r11 // ... mod 4 + subq $1,%rax // nr-- + pxor %xmm3,%xmm0 // 0 = 2A+3B+C+D pxor %xmm11,%xmm6 .Lenc2x_entry: - - movdqa %xmm9,%xmm1 +// top of round + movdqa %xmm9,%xmm1 // 1 : i movdqa %xmm9,%xmm7 - movdqa .Lk_inv+16(%rip),%xmm5 + movdqa .Lk_inv+16(%rip),%xmm5 // 2 : a/k movdqa %xmm5,%xmm13 - pandn %xmm0,%xmm1 + pandn %xmm0,%xmm1 // 1 = i<<4 pandn %xmm6,%xmm7 - psrld $4,%xmm1 + psrld $4,%xmm1 // 1 = i psrld $4,%xmm7 - pand %xmm9,%xmm0 + pand %xmm9,%xmm0 // 0 = k pand %xmm9,%xmm6 - pshufb %xmm0,%xmm5 + pshufb %xmm0,%xmm5 // 2 = a/k pshufb %xmm6,%xmm13 - movdqa %xmm10,%xmm3 + movdqa %xmm10,%xmm3 // 3 : 1/i movdqa %xmm10,%xmm11 - pxor %xmm1,%xmm0 + pxor %xmm1,%xmm0 // 0 = j pxor %xmm7,%xmm6 - pshufb %xmm1,%xmm3 + pshufb %xmm1,%xmm3 // 3 = 1/i pshufb %xmm7,%xmm11 - movdqa %xmm10,%xmm4 + movdqa %xmm10,%xmm4 // 4 : 1/j movdqa %xmm10,%xmm12 - pxor %xmm5,%xmm3 + pxor %xmm5,%xmm3 // 3 = iak = 1/i + a/k pxor %xmm13,%xmm11 - pshufb %xmm0,%xmm4 + pshufb %xmm0,%xmm4 // 4 = 1/j pshufb %xmm6,%xmm12 - movdqa %xmm10,%xmm2 + movdqa %xmm10,%xmm2 // 2 : 1/iak movdqa %xmm10,%xmm8 - pxor %xmm5,%xmm4 + pxor %xmm5,%xmm4 // 4 = jak = 1/j + a/k pxor %xmm13,%xmm12 - pshufb %xmm3,%xmm2 + pshufb %xmm3,%xmm2 // 2 = 1/iak pshufb %xmm11,%xmm8 - movdqa %xmm10,%xmm3 + movdqa %xmm10,%xmm3 // 3 : 1/jak movdqa %xmm10,%xmm11 - pxor %xmm0,%xmm2 + pxor %xmm0,%xmm2 // 2 = io pxor %xmm6,%xmm8 - pshufb %xmm4,%xmm3 + pshufb %xmm4,%xmm3 // 3 = 1/jak pshufb %xmm12,%xmm11 movdqu (%r9),%xmm5 - - pxor %xmm1,%xmm3 +// Also use %xmm5 in the second instance. + pxor %xmm1,%xmm3 // 3 = jo pxor %xmm7,%xmm11 jnz .Lenc2x_loop - - movdqa -96(%r10),%xmm4 - movdqa -80(%r10),%xmm0 +// middle of last round + movdqa -96(%r10),%xmm4 // 3 : sbou .Lk_sbo + movdqa -80(%r10),%xmm0 // 0 : sbot .Lk_sbo+16 movdqa %xmm4,%xmm12 movdqa %xmm0,%xmm6 - pshufb %xmm2,%xmm4 + pshufb %xmm2,%xmm4 // 4 = sbou pshufb %xmm8,%xmm12 - pxor %xmm5,%xmm4 + pxor %xmm5,%xmm4 // 4 = sb1u + k pxor %xmm5,%xmm12 - pshufb %xmm3,%xmm0 + pshufb %xmm3,%xmm0 // 0 = sb1t pshufb %xmm11,%xmm6 - movdqa 64(%r11,%r10,1),%xmm1 - - pxor %xmm4,%xmm0 + movdqa 64(%r11,%r10,1),%xmm1 // .Lk_sr[] +// Also use %xmm1 in the second instance. + pxor %xmm4,%xmm0 // 0 = A pxor %xmm12,%xmm6 pshufb %xmm1,%xmm0 pshufb %xmm1,%xmm6 @@ -283,27 +283,27 @@ .cfi_endproc .size _vpaes_encrypt_core_2x,.-_vpaes_encrypt_core_2x - - - - - +// +// Decryption core +// +// Same API as encryption core. +// .type _vpaes_decrypt_core,@function .align 16 _vpaes_decrypt_core: .cfi_startproc - movq %rdx,%r9 + movq %rdx,%r9 // load key movl 240(%rdx),%eax movdqa %xmm9,%xmm1 - movdqa .Lk_dipt(%rip),%xmm2 + movdqa .Lk_dipt(%rip),%xmm2 // iptlo pandn %xmm0,%xmm1 movq %rax,%r11 psrld $4,%xmm1 - movdqu (%r9),%xmm5 + movdqu (%r9),%xmm5 // round0 key shlq $4,%r11 pand %xmm9,%xmm0 pshufb %xmm0,%xmm2 - movdqa .Lk_dipt+16(%rip),%xmm0 + movdqa .Lk_dipt+16(%rip),%xmm0 // ipthi xorq $0x30,%r11 leaq .Lk_dsbd(%rip),%r10 pshufb %xmm1,%xmm0 @@ -317,99 +317,99 @@ .align 16 .Ldec_loop: - - - - movdqa -32(%r10),%xmm4 - movdqa -16(%r10),%xmm1 - pshufb %xmm2,%xmm4 - pshufb %xmm3,%xmm1 +// +// Inverse mix columns +// + movdqa -32(%r10),%xmm4 // 4 : sb9u + movdqa -16(%r10),%xmm1 // 0 : sb9t + pshufb %xmm2,%xmm4 // 4 = sb9u + pshufb %xmm3,%xmm1 // 0 = sb9t pxor %xmm4,%xmm0 - movdqa 0(%r10),%xmm4 - pxor %xmm1,%xmm0 - movdqa 16(%r10),%xmm1 + movdqa 0(%r10),%xmm4 // 4 : sbdu + pxor %xmm1,%xmm0 // 0 = ch + movdqa 16(%r10),%xmm1 // 0 : sbdt - pshufb %xmm2,%xmm4 - pshufb %xmm5,%xmm0 - pshufb %xmm3,%xmm1 - pxor %xmm4,%xmm0 - movdqa 32(%r10),%xmm4 - pxor %xmm1,%xmm0 - movdqa 48(%r10),%xmm1 + pshufb %xmm2,%xmm4 // 4 = sbdu + pshufb %xmm5,%xmm0 // MC ch + pshufb %xmm3,%xmm1 // 0 = sbdt + pxor %xmm4,%xmm0 // 4 = ch + movdqa 32(%r10),%xmm4 // 4 : sbbu + pxor %xmm1,%xmm0 // 0 = ch + movdqa 48(%r10),%xmm1 // 0 : sbbt - pshufb %xmm2,%xmm4 - pshufb %xmm5,%xmm0 - pshufb %xmm3,%xmm1 - pxor %xmm4,%xmm0 - movdqa 64(%r10),%xmm4 - pxor %xmm1,%xmm0 - movdqa 80(%r10),%xmm1 + pshufb %xmm2,%xmm4 // 4 = sbbu + pshufb %xmm5,%xmm0 // MC ch + pshufb %xmm3,%xmm1 // 0 = sbbt + pxor %xmm4,%xmm0 // 4 = ch + movdqa 64(%r10),%xmm4 // 4 : sbeu + pxor %xmm1,%xmm0 // 0 = ch + movdqa 80(%r10),%xmm1 // 0 : sbet - pshufb %xmm2,%xmm4 - pshufb %xmm5,%xmm0 - pshufb %xmm3,%xmm1 - pxor %xmm4,%xmm0 - addq $16,%r9 + pshufb %xmm2,%xmm4 // 4 = sbeu + pshufb %xmm5,%xmm0 // MC ch + pshufb %xmm3,%xmm1 // 0 = sbet + pxor %xmm4,%xmm0 // 4 = ch + addq $16,%r9 // next round key palignr $12,%xmm5,%xmm5 - pxor %xmm1,%xmm0 - subq $1,%rax + pxor %xmm1,%xmm0 // 0 = ch + subq $1,%rax // nr-- .Ldec_entry: - - movdqa %xmm9,%xmm1 - pandn %xmm0,%xmm1 - movdqa %xmm11,%xmm2 - psrld $4,%xmm1 - pand %xmm9,%xmm0 - pshufb %xmm0,%xmm2 - movdqa %xmm10,%xmm3 - pxor %xmm1,%xmm0 - pshufb %xmm1,%xmm3 - movdqa %xmm10,%xmm4 - pxor %xmm2,%xmm3 - pshufb %xmm0,%xmm4 - pxor %xmm2,%xmm4 - movdqa %xmm10,%xmm2 - pshufb %xmm3,%xmm2 - movdqa %xmm10,%xmm3 - pxor %xmm0,%xmm2 - pshufb %xmm4,%xmm3 +// top of round + movdqa %xmm9,%xmm1 // 1 : i + pandn %xmm0,%xmm1 // 1 = i<<4 + movdqa %xmm11,%xmm2 // 2 : a/k + psrld $4,%xmm1 // 1 = i + pand %xmm9,%xmm0 // 0 = k + pshufb %xmm0,%xmm2 // 2 = a/k + movdqa %xmm10,%xmm3 // 3 : 1/i + pxor %xmm1,%xmm0 // 0 = j + pshufb %xmm1,%xmm3 // 3 = 1/i + movdqa %xmm10,%xmm4 // 4 : 1/j + pxor %xmm2,%xmm3 // 3 = iak = 1/i + a/k + pshufb %xmm0,%xmm4 // 4 = 1/j + pxor %xmm2,%xmm4 // 4 = jak = 1/j + a/k + movdqa %xmm10,%xmm2 // 2 : 1/iak + pshufb %xmm3,%xmm2 // 2 = 1/iak + movdqa %xmm10,%xmm3 // 3 : 1/jak + pxor %xmm0,%xmm2 // 2 = io + pshufb %xmm4,%xmm3 // 3 = 1/jak movdqu (%r9),%xmm0 - pxor %xmm1,%xmm3 + pxor %xmm1,%xmm3 // 3 = jo jnz .Ldec_loop - - movdqa 96(%r10),%xmm4 - pshufb %xmm2,%xmm4 - pxor %xmm0,%xmm4 - movdqa 112(%r10),%xmm0 - movdqa -352(%r11),%xmm2 - pshufb %xmm3,%xmm0 - pxor %xmm4,%xmm0 +// middle of last round + movdqa 96(%r10),%xmm4 // 3 : sbou + pshufb %xmm2,%xmm4 // 4 = sbou + pxor %xmm0,%xmm4 // 4 = sb1u + k + movdqa 112(%r10),%xmm0 // 0 : sbot + movdqa -352(%r11),%xmm2 // .Lk_sr-.Lk_dsbd=-0x160 + pshufb %xmm3,%xmm0 // 0 = sb1t + pxor %xmm4,%xmm0 // 0 = A pshufb %xmm2,%xmm0 ret .cfi_endproc .size _vpaes_decrypt_core,.-_vpaes_decrypt_core - - - - - +// ###################################################### +// ## +// AES key schedule ## +// ## +// ###################################################### .type _vpaes_schedule_core,@function .align 16 _vpaes_schedule_core: .cfi_startproc +// rdi = key +// rsi = size in bits +// rdx = buffer +// rcx = direction. 0=encrypt, 1=decrypt + call _vpaes_preheat // load the tables + movdqa .Lk_rcon(%rip),%xmm8 // load rcon + movdqu (%rdi),%xmm0 // load key (unaligned) - - - - call _vpaes_preheat - movdqa .Lk_rcon(%rip),%xmm8 - movdqu (%rdi),%xmm0 - - +// input transform movdqa %xmm0,%xmm3 leaq .Lk_ipt(%rip),%r11 call _vpaes_schedule_transform @@ -419,12 +419,12 @@ testq %rcx,%rcx jnz .Lschedule_am_decrypting - +// encrypting, output zeroth round key after transform movdqu %xmm0,(%rdx) jmp .Lschedule_go .Lschedule_am_decrypting: - +// decrypting, output zeroth round key after shiftrows movdqa (%r8,%r10,1),%xmm1 pshufb %xmm1,%xmm3 movdqu %xmm3,(%rdx) @@ -434,16 +434,16 @@ cmpl $192,%esi ja .Lschedule_256 je .Lschedule_192 +// 128: fall though - - - - - - - - - +// +// .schedule_128 +// +// 128-bit specific part of key schedule. +// +// This schedule is really simple, because all its parts +// are accomplished by the subroutines. +// .Lschedule_128: movl $10,%esi @@ -451,73 +451,73 @@ call _vpaes_schedule_round decq %rsi jz .Lschedule_mangle_last - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // write output jmp .Loop_schedule_128 - - - - - - - - - - - - - - - +// +// .aes_schedule_192 +// +// 192-bit specific part of key schedule. +// +// The main body of this schedule is the same as the 128-bit +// schedule, but with more smearing. The long, high side is +// stored in %xmm7 as before, and the short, low side is in +// the high bits of %xmm6. +// +// This schedule is somewhat nastier, however, because each +// round produces 192 bits of key material, or 1.5 round keys. +// Therefore, on each cycle we do 2 rounds and produce 3 round +// keys. +// .align 16 .Lschedule_192: - movdqu 8(%rdi),%xmm0 - call _vpaes_schedule_transform - movdqa %xmm0,%xmm6 - pxor %xmm4,%xmm4 - movhlps %xmm4,%xmm6 + movdqu 8(%rdi),%xmm0 // load key part 2 (very unaligned) + call _vpaes_schedule_transform // input transform + movdqa %xmm0,%xmm6 // save short part + pxor %xmm4,%xmm4 // clear 4 + movhlps %xmm4,%xmm6 // clobber low side with zeros movl $4,%esi .Loop_schedule_192: call _vpaes_schedule_round palignr $8,%xmm6,%xmm0 - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // save key n call _vpaes_schedule_192_smear - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // save key n+1 call _vpaes_schedule_round decq %rsi jz .Lschedule_mangle_last - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle // save key n+2 call _vpaes_schedule_192_smear jmp .Loop_schedule_192 - - - - - - - - - - +// +// .aes_schedule_256 +// +// 256-bit specific part of key schedule. +// +// The structure here is very similar to the 128-bit +// schedule, but with an additional "low side" in +// %xmm6. The low side's rounds are the same as the +// high side's, except no rcon and no rotation. +// .align 16 .Lschedule_256: - movdqu 16(%rdi),%xmm0 - call _vpaes_schedule_transform + movdqu 16(%rdi),%xmm0 // load key part 2 (unaligned) + call _vpaes_schedule_transform // input transform movl $7,%esi .Loop_schedule_256: - call _vpaes_schedule_mangle - movdqa %xmm0,%xmm6 + call _vpaes_schedule_mangle // output low result + movdqa %xmm0,%xmm6 // save cur_lo in xmm6 - +// high round call _vpaes_schedule_round decq %rsi jz .Lschedule_mangle_last call _vpaes_schedule_mangle - +// low round. swap xmm7 and xmm6 pshufd $0xFF,%xmm0,%xmm0 movdqa %xmm7,%xmm5 movdqa %xmm6,%xmm7 @@ -527,36 +527,36 @@ jmp .Loop_schedule_256 - - - - - - - - - - +// +// .aes_schedule_mangle_last +// +// Mangler for last round of key schedule +// Mangles %xmm0 +// when encrypting, outputs out(%xmm0) ^ 63 +// when decrypting, outputs unskew(%xmm0) +// +// Always called right before return... jumps to cleanup and exits +// .align 16 .Lschedule_mangle_last: - - leaq .Lk_deskew(%rip),%r11 +// schedule last round key from xmm0 + leaq .Lk_deskew(%rip),%r11 // prepare to deskew testq %rcx,%rcx jnz .Lschedule_mangle_last_dec - +// encrypting movdqa (%r8,%r10,1),%xmm1 - pshufb %xmm1,%xmm0 - leaq .Lk_opt(%rip),%r11 + pshufb %xmm1,%xmm0 // output permute + leaq .Lk_opt(%rip),%r11 // prepare to output transform addq $32,%rdx .Lschedule_mangle_last_dec: addq $-16,%rdx pxor .Lk_s63(%rip),%xmm0 - call _vpaes_schedule_transform - movdqu %xmm0,(%rdx) + call _vpaes_schedule_transform // output transform + movdqu %xmm0,(%rdx) // save last key - +// cleanup pxor %xmm0,%xmm0 pxor %xmm1,%xmm1 pxor %xmm2,%xmm2 @@ -569,72 +569,72 @@ .cfi_endproc .size _vpaes_schedule_core,.-_vpaes_schedule_core - - - - - - - - - - - - - - +// +// .aes_schedule_192_smear +// +// Smear the short, low side in the 192-bit key schedule. +// +// Inputs: +// %xmm7: high side, b a x y +// %xmm6: low side, d c 0 0 +// %xmm13: 0 +// +// Outputs: +// %xmm6: b+c+d b+c 0 0 +// %xmm0: b+c+d b+c b a +// .type _vpaes_schedule_192_smear,@function .align 16 _vpaes_schedule_192_smear: .cfi_startproc - pshufd $0x80,%xmm6,%xmm1 - pshufd $0xFE,%xmm7,%xmm0 - pxor %xmm1,%xmm6 + pshufd $0x80,%xmm6,%xmm1 // d c 0 0 -> c 0 0 0 + pshufd $0xFE,%xmm7,%xmm0 // b a _ _ -> b b b a + pxor %xmm1,%xmm6 // -> c+d c 0 0 pxor %xmm1,%xmm1 - pxor %xmm0,%xmm6 + pxor %xmm0,%xmm6 // -> b+c+d b+c b a movdqa %xmm6,%xmm0 - movhlps %xmm1,%xmm6 + movhlps %xmm1,%xmm6 // clobber low side with zeros ret .cfi_endproc .size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear - - - - - - - - - - - - - - - - - - +// +// .aes_schedule_round +// +// Runs one main round of the key schedule on %xmm0, %xmm7 +// +// Specifically, runs subbytes on the high dword of %xmm0 +// then rotates it by one byte and xors into the low dword of +// %xmm7. +// +// Adds rcon from low byte of %xmm8, then rotates %xmm8 for +// next rcon. +// +// Smears the dwords of %xmm7 by xoring the low into the +// second low, result into third, result into highest. +// +// Returns results in %xmm7 = %xmm0. +// Clobbers %xmm1-%xmm4, %r11. +// .type _vpaes_schedule_round,@function .align 16 _vpaes_schedule_round: .cfi_startproc - +// extract rcon from xmm8 pxor %xmm1,%xmm1 palignr $15,%xmm8,%xmm1 palignr $15,%xmm8,%xmm8 pxor %xmm1,%xmm7 - +// rotate pshufd $0xFF,%xmm0,%xmm0 palignr $1,%xmm0,%xmm0 +// fall through... - - +// low round: same as high round, but no rotation and no rcon. _vpaes_schedule_low_round: - +// smear xmm7 movdqa %xmm7,%xmm1 pslldq $4,%xmm7 pxor %xmm1,%xmm7 @@ -643,48 +643,48 @@ pxor %xmm1,%xmm7 pxor .Lk_s63(%rip),%xmm7 - +// subbytes movdqa %xmm9,%xmm1 pandn %xmm0,%xmm1 - psrld $4,%xmm1 - pand %xmm9,%xmm0 - movdqa %xmm11,%xmm2 - pshufb %xmm0,%xmm2 - pxor %xmm1,%xmm0 - movdqa %xmm10,%xmm3 - pshufb %xmm1,%xmm3 - pxor %xmm2,%xmm3 - movdqa %xmm10,%xmm4 - pshufb %xmm0,%xmm4 - pxor %xmm2,%xmm4 - movdqa %xmm10,%xmm2 - pshufb %xmm3,%xmm2 - pxor %xmm0,%xmm2 - movdqa %xmm10,%xmm3 - pshufb %xmm4,%xmm3 - pxor %xmm1,%xmm3 - movdqa %xmm13,%xmm4 - pshufb %xmm2,%xmm4 - movdqa %xmm12,%xmm0 - pshufb %xmm3,%xmm0 - pxor %xmm4,%xmm0 + psrld $4,%xmm1 // 1 = i + pand %xmm9,%xmm0 // 0 = k + movdqa %xmm11,%xmm2 // 2 : a/k + pshufb %xmm0,%xmm2 // 2 = a/k + pxor %xmm1,%xmm0 // 0 = j + movdqa %xmm10,%xmm3 // 3 : 1/i + pshufb %xmm1,%xmm3 // 3 = 1/i + pxor %xmm2,%xmm3 // 3 = iak = 1/i + a/k + movdqa %xmm10,%xmm4 // 4 : 1/j + pshufb %xmm0,%xmm4 // 4 = 1/j + pxor %xmm2,%xmm4 // 4 = jak = 1/j + a/k + movdqa %xmm10,%xmm2 // 2 : 1/iak + pshufb %xmm3,%xmm2 // 2 = 1/iak + pxor %xmm0,%xmm2 // 2 = io + movdqa %xmm10,%xmm3 // 3 : 1/jak + pshufb %xmm4,%xmm3 // 3 = 1/jak + pxor %xmm1,%xmm3 // 3 = jo + movdqa %xmm13,%xmm4 // 4 : sbou + pshufb %xmm2,%xmm4 // 4 = sbou + movdqa %xmm12,%xmm0 // 0 : sbot + pshufb %xmm3,%xmm0 // 0 = sb1t + pxor %xmm4,%xmm0 // 0 = sbox output - +// add in smeared stuff pxor %xmm7,%xmm0 movdqa %xmm0,%xmm7 ret .cfi_endproc .size _vpaes_schedule_round,.-_vpaes_schedule_round - - - - - - - - - +// +// .aes_schedule_transform +// +// Linear-transform %xmm0 according to tables at (%r11) +// +// Requires that %xmm9 = 0x0F0F... as in preheat +// Output in %xmm0 +// Clobbers %xmm1, %xmm2 +// .type _vpaes_schedule_transform,@function .align 16 _vpaes_schedule_transform: @@ -693,48 +693,48 @@ pandn %xmm0,%xmm1 psrld $4,%xmm1 pand %xmm9,%xmm0 - movdqa (%r11),%xmm2 + movdqa (%r11),%xmm2 // lo pshufb %xmm0,%xmm2 - movdqa 16(%r11),%xmm0 + movdqa 16(%r11),%xmm0 // hi pshufb %xmm1,%xmm0 pxor %xmm2,%xmm0 ret .cfi_endproc .size _vpaes_schedule_transform,.-_vpaes_schedule_transform - - - - - - - - - - - - - - - - - - - - - - - +// +// .aes_schedule_mangle +// +// Mangle xmm0 from (basis-transformed) standard version +// to our version. +// +// On encrypt, +// xor with 0x63 +// multiply by circulant 0,1,1,1 +// apply shiftrows transform +// +// On decrypt, +// xor with 0x63 +// multiply by "inverse mixcolumns" circulant E,B,D,9 +// deskew +// apply shiftrows transform +// +// +// Writes out to (%rdx), and increments or decrements it +// Keeps track of round number mod 4 in %r8 +// Preserves xmm0 +// Clobbers xmm1-xmm5 +// .type _vpaes_schedule_mangle,@function .align 16 _vpaes_schedule_mangle: .cfi_startproc - movdqa %xmm0,%xmm4 + movdqa %xmm0,%xmm4 // save xmm0 for later movdqa .Lk_mc_forward(%rip),%xmm5 testq %rcx,%rcx jnz .Lschedule_mangle_dec - +// encrypting addq $16,%rdx pxor .Lk_s63(%rip),%xmm4 pshufb %xmm5,%xmm4 @@ -747,12 +747,12 @@ jmp .Lschedule_mangle_both .align 16 .Lschedule_mangle_dec: - +// inverse mix columns leaq .Lk_dksd(%rip),%r11 movdqa %xmm9,%xmm1 pandn %xmm4,%xmm1 - psrld $4,%xmm1 - pand %xmm9,%xmm4 + psrld $4,%xmm1 // 1 = hi + pand %xmm9,%xmm4 // 4 = lo movdqa 0(%r11),%xmm2 pshufb %xmm4,%xmm2 @@ -796,9 +796,9 @@ .cfi_endproc .size _vpaes_schedule_mangle,.-_vpaes_schedule_mangle - - - +// +// Interface to OpenSSL +// .globl vpaes_set_encrypt_key .hidden vpaes_set_encrypt_key .type vpaes_set_encrypt_key,@function @@ -815,7 +815,7 @@ movl %esi,%eax shrl $5,%eax addl $5,%eax - movl %eax,240(%rdx) + movl %eax,240(%rdx) // AES_KEY->rounds = nbits/32+5; movl $0,%ecx movl $0x30,%r8d @@ -835,7 +835,7 @@ movl %esi,%eax shrl $5,%eax addl $5,%eax - movl %eax,240(%rdx) + movl %eax,240(%rdx) // AES_KEY->rounds = nbits/32+5; shll $4,%eax leaq 16(%rdx,%rax,1),%rdx @@ -843,7 +843,7 @@ movl %esi,%r8d shrl $1,%r8d andl $32,%r8d - xorl $32,%r8d + xorl $32,%r8d // nbits==192?0:32 call _vpaes_schedule_core xorl %eax,%eax ret @@ -894,7 +894,7 @@ xchgq %rcx,%rdx subq $16,%rcx jc .Lcbc_abort - movdqu (%r8),%xmm6 + movdqu (%r8),%xmm6 // load IV subq %rdi,%rsi call _vpaes_preheat cmpl $0,%r9d @@ -923,7 +923,7 @@ subq $16,%rcx jnc .Lcbc_dec_loop .Lcbc_done: - movdqu %xmm6,(%r8) + movdqu %xmm6,(%r8) // save IV .Lcbc_abort: ret .cfi_endproc @@ -935,13 +935,13 @@ vpaes_ctr32_encrypt_blocks: .cfi_startproc _CET_ENDBR - +// _vpaes_encrypt_core and _vpaes_encrypt_core_2x expect the key in %rdx. xchgq %rcx,%rdx testq %rcx,%rcx jz .Lctr32_abort - movdqu (%r8),%xmm0 + movdqu (%r8),%xmm0 // Load IV. movdqa .Lctr_add_one(%rip),%xmm8 - subq %rdi,%rsi + subq %rdi,%rsi // This allows only incrementing %rdi. call _vpaes_preheat movdqa %xmm0,%xmm6 pshufb .Lrev_ctr(%rip),%xmm6 @@ -949,9 +949,9 @@ testq $1,%rcx jz .Lctr32_prep_loop - - - movdqu (%rdi),%xmm7 +// Handle one block so the remaining block count is even for +// _vpaes_encrypt_core_2x. + movdqu (%rdi),%xmm7 // Load input. call _vpaes_encrypt_core pxor %xmm7,%xmm0 paddd %xmm8,%xmm6 @@ -961,29 +961,29 @@ jz .Lctr32_done .Lctr32_prep_loop: - - +// _vpaes_encrypt_core_2x leaves only %xmm14 and %xmm15 as spare +// registers. We maintain two byte-swapped counters in them. movdqa %xmm6,%xmm14 movdqa %xmm6,%xmm15 paddd %xmm8,%xmm15 .Lctr32_loop: - movdqa .Lrev_ctr(%rip),%xmm1 + movdqa .Lrev_ctr(%rip),%xmm1 // Set up counters. movdqa %xmm14,%xmm0 movdqa %xmm15,%xmm6 pshufb %xmm1,%xmm0 pshufb %xmm1,%xmm6 call _vpaes_encrypt_core_2x - movdqu (%rdi),%xmm1 + movdqu (%rdi),%xmm1 // Load input. movdqu 16(%rdi),%xmm2 movdqa .Lctr_add_two(%rip),%xmm3 - pxor %xmm1,%xmm0 + pxor %xmm1,%xmm0 // XOR input. pxor %xmm2,%xmm6 - paddd %xmm3,%xmm14 + paddd %xmm3,%xmm14 // Increment counters. paddd %xmm3,%xmm15 - movdqu %xmm0,(%rsi,%rdi,1) + movdqu %xmm0,(%rsi,%rdi,1) // Write output. movdqu %xmm6,16(%rsi,%rdi,1) - subq $2,%rcx + subq $2,%rcx // Advance loop. leaq 32(%rdi),%rdi jnz .Lctr32_loop @@ -992,135 +992,135 @@ ret .cfi_endproc .size vpaes_ctr32_encrypt_blocks,.-vpaes_ctr32_encrypt_blocks - - - - - - +// +// _aes_preheat +// +// Fills register %r10 -> .aes_consts (so you can -fPIC) +// and %xmm9-%xmm15 as specified below. +// .type _vpaes_preheat,@function .align 16 _vpaes_preheat: .cfi_startproc leaq .Lk_s0F(%rip),%r10 - movdqa -32(%r10),%xmm10 - movdqa -16(%r10),%xmm11 - movdqa 0(%r10),%xmm9 - movdqa 48(%r10),%xmm13 - movdqa 64(%r10),%xmm12 - movdqa 80(%r10),%xmm15 - movdqa 96(%r10),%xmm14 + movdqa -32(%r10),%xmm10 // .Lk_inv + movdqa -16(%r10),%xmm11 // .Lk_inv+16 + movdqa 0(%r10),%xmm9 // .Lk_s0F + movdqa 48(%r10),%xmm13 // .Lk_sb1 + movdqa 64(%r10),%xmm12 // .Lk_sb1+16 + movdqa 80(%r10),%xmm15 // .Lk_sb2 + movdqa 96(%r10),%xmm14 // .Lk_sb2+16 ret .cfi_endproc .size _vpaes_preheat,.-_vpaes_preheat - - - - - +// ###################################################### +// ## +// Constants ## +// ## +// ###################################################### .type _vpaes_consts,@object .section .rodata .align 64 _vpaes_consts: -.Lk_inv: +.Lk_inv: // inv, inva .quad 0x0E05060F0D080180, 0x040703090A0B0C02 .quad 0x01040A060F0B0780, 0x030D0E0C02050809 -.Lk_s0F: +.Lk_s0F: // s0F .quad 0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F0F0F0F0F -.Lk_ipt: +.Lk_ipt: // input transform (lo, hi) .quad 0xC2B2E8985A2A7000, 0xCABAE09052227808 .quad 0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81 -.Lk_sb1: +.Lk_sb1: // sb1u, sb1t .quad 0xB19BE18FCB503E00, 0xA5DF7A6E142AF544 .quad 0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF -.Lk_sb2: +.Lk_sb2: // sb2u, sb2t .quad 0xE27A93C60B712400, 0x5EB7E955BC982FCD .quad 0x69EB88400AE12900, 0xC2A163C8AB82234A -.Lk_sbo: +.Lk_sbo: // sbou, sbot .quad 0xD0D26D176FBDC700, 0x15AABF7AC502A878 .quad 0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA -.Lk_mc_forward: +.Lk_mc_forward: // mc_forward .quad 0x0407060500030201, 0x0C0F0E0D080B0A09 .quad 0x080B0A0904070605, 0x000302010C0F0E0D .quad 0x0C0F0E0D080B0A09, 0x0407060500030201 .quad 0x000302010C0F0E0D, 0x080B0A0904070605 -.Lk_mc_backward: +.Lk_mc_backward: // mc_backward .quad 0x0605040702010003, 0x0E0D0C0F0A09080B .quad 0x020100030E0D0C0F, 0x0A09080B06050407 .quad 0x0E0D0C0F0A09080B, 0x0605040702010003 .quad 0x0A09080B06050407, 0x020100030E0D0C0F -.Lk_sr: +.Lk_sr: // sr .quad 0x0706050403020100, 0x0F0E0D0C0B0A0908 .quad 0x030E09040F0A0500, 0x0B06010C07020D08 .quad 0x0F060D040B020900, 0x070E050C030A0108 .quad 0x0B0E0104070A0D00, 0x0306090C0F020508 -.Lk_rcon: +.Lk_rcon: // rcon .quad 0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81 -.Lk_s63: +.Lk_s63: // s63: all equal to 0x63 transformed .quad 0x5B5B5B5B5B5B5B5B, 0x5B5B5B5B5B5B5B5B -.Lk_opt: +.Lk_opt: // output transform .quad 0xFF9F4929D6B66000, 0xF7974121DEBE6808 .quad 0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0 -.Lk_deskew: +.Lk_deskew: // deskew tables: inverts the sbox's "skew" .quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A .quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77 - - - - -.Lk_dksd: +// +// Decryption stuff +// Key schedule constants +// +.Lk_dksd: // decryption key schedule: invskew x*D .quad 0xFEB91A5DA3E44700, 0x0740E3A45A1DBEF9 .quad 0x41C277F4B5368300, 0x5FDC69EAAB289D1E -.Lk_dksb: +.Lk_dksb: // decryption key schedule: invskew x*B .quad 0x9A4FCA1F8550D500, 0x03D653861CC94C99 .quad 0x115BEDA7B6FC4A00, 0xD993256F7E3482C8 -.Lk_dkse: +.Lk_dkse: // decryption key schedule: invskew x*E + 0x63 .quad 0xD5031CCA1FC9D600, 0x53859A4C994F5086 .quad 0xA23196054FDC7BE8, 0xCD5EF96A20B31487 -.Lk_dks9: +.Lk_dks9: // decryption key schedule: invskew x*9 .quad 0xB6116FC87ED9A700, 0x4AED933482255BFC .quad 0x4576516227143300, 0x8BB89FACE9DAFDCE - - - - -.Lk_dipt: +// +// Decryption stuff +// Round function constants +// +.Lk_dipt: // decryption input transform .quad 0x0F505B040B545F00, 0x154A411E114E451A .quad 0x86E383E660056500, 0x12771772F491F194 -.Lk_dsb9: +.Lk_dsb9: // decryption sbox output *9*u, *9*t .quad 0x851C03539A86D600, 0xCAD51F504F994CC9 .quad 0xC03B1789ECD74900, 0x725E2C9EB2FBA565 -.Lk_dsbd: +.Lk_dsbd: // decryption sbox output *D*u, *D*t .quad 0x7D57CCDFE6B1A200, 0xF56E9B13882A4439 .quad 0x3CE2FAF724C6CB00, 0x2931180D15DEEFD3 -.Lk_dsbb: +.Lk_dsbb: // decryption sbox output *B*u, *B*t .quad 0xD022649296B44200, 0x602646F6B0F2D404 .quad 0xC19498A6CD596700, 0xF3FF0C3E3255AA6B -.Lk_dsbe: +.Lk_dsbe: // decryption sbox output *E*u, *E*t .quad 0x46F2929626D4D000, 0x2242600464B4F6B0 .quad 0x0C55A6CDFFAAC100, 0x9467F36B98593E32 -.Lk_dsbo: +.Lk_dsbo: // decryption sbox final output .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C - +// .Lrev_ctr is a permutation which byte-swaps the counter portion of the IV. .Lrev_ctr: .quad 0x0706050403020100, 0x0c0d0e0f0b0a0908 - - +// .Lctr_add_* may be added to a byte-swapped xmm register to increment the +// counter. The register must be byte-swapped again to form the actual input. .Lctr_add_one: .quad 0x0000000000000000, 0x0000000100000000 .Lctr_add_two:
diff --git a/gen/bcm/vpaes-x86_64-win.asm b/gen/bcm/vpaes-x86_64-win.asm index cddb9fa..001618c 100644 --- a/gen/bcm/vpaes-x86_64-win.asm +++ b/gen/bcm/vpaes-x86_64-win.asm
@@ -14,21 +14,21 @@ section .text code align=64 - - - - - - - - - - - - - - - +; # +; # _aes_encrypt_core +; # +; # AES-encrypt %xmm0. +; # +; # Inputs: +; # %xmm0 = input +; # %xmm9-%xmm15 as in _vpaes_preheat +; # (%rdx) = scheduled keys +; # +; # Output in %xmm0 +; # Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax +; # Preserves %xmm6 - %xmm8 so you get some local vectors +; # +; # ALIGN 16 _vpaes_encrypt_core: @@ -37,13 +37,13 @@ mov r11,16 mov eax,DWORD[240+rdx] movdqa xmm1,xmm9 - movdqa xmm2,XMMWORD[$L$k_ipt] + movdqa xmm2,XMMWORD[$L$k_ipt] ; iptlo pandn xmm1,xmm0 - movdqu xmm5,XMMWORD[r9] + movdqu xmm5,XMMWORD[r9] ; round0 key psrld xmm1,4 pand xmm0,xmm9 pshufb xmm2,xmm0 - movdqa xmm0,XMMWORD[(($L$k_ipt+16))] + movdqa xmm0,XMMWORD[(($L$k_ipt+16))] ; ipthi pshufb xmm0,xmm1 pxor xmm2,xmm5 add r9,16 @@ -53,98 +53,98 @@ ALIGN 16 $L$enc_loop: - - movdqa xmm4,xmm13 - movdqa xmm0,xmm12 - pshufb xmm4,xmm2 - pshufb xmm0,xmm3 - pxor xmm4,xmm5 - movdqa xmm5,xmm15 - pxor xmm0,xmm4 - movdqa xmm1,XMMWORD[((-64))+r10*1+r11] - pshufb xmm5,xmm2 - movdqa xmm4,XMMWORD[r10*1+r11] - movdqa xmm2,xmm14 - pshufb xmm2,xmm3 - movdqa xmm3,xmm0 - pxor xmm2,xmm5 - pshufb xmm0,xmm1 - add r9,16 - pxor xmm0,xmm2 - pshufb xmm3,xmm4 - add r11,16 - pxor xmm3,xmm0 - pshufb xmm0,xmm1 - and r11,0x30 - sub rax,1 - pxor xmm0,xmm3 +; middle of middle round + movdqa xmm4,xmm13 ; 4 : sb1u + movdqa xmm0,xmm12 ; 0 : sb1t + pshufb xmm4,xmm2 ; 4 = sb1u + pshufb xmm0,xmm3 ; 0 = sb1t + pxor xmm4,xmm5 ; 4 = sb1u + k + movdqa xmm5,xmm15 ; 4 : sb2u + pxor xmm0,xmm4 ; 0 = A + movdqa xmm1,XMMWORD[((-64))+r10*1+r11] ; .Lk_mc_forward[] + pshufb xmm5,xmm2 ; 4 = sb2u + movdqa xmm4,XMMWORD[r10*1+r11] ; .Lk_mc_backward[] + movdqa xmm2,xmm14 ; 2 : sb2t + pshufb xmm2,xmm3 ; 2 = sb2t + movdqa xmm3,xmm0 ; 3 = A + pxor xmm2,xmm5 ; 2 = 2A + pshufb xmm0,xmm1 ; 0 = B + add r9,16 ; next key + pxor xmm0,xmm2 ; 0 = 2A+B + pshufb xmm3,xmm4 ; 3 = D + add r11,16 ; next mc + pxor xmm3,xmm0 ; 3 = 2A+B+D + pshufb xmm0,xmm1 ; 0 = 2B+C + and r11,0x30 ; ... mod 4 + sub rax,1 ; nr-- + pxor xmm0,xmm3 ; 0 = 2A+3B+C+D $L$enc_entry: - - movdqa xmm1,xmm9 - movdqa xmm5,xmm11 - pandn xmm1,xmm0 - psrld xmm1,4 - pand xmm0,xmm9 - pshufb xmm5,xmm0 - movdqa xmm3,xmm10 - pxor xmm0,xmm1 - pshufb xmm3,xmm1 - movdqa xmm4,xmm10 - pxor xmm3,xmm5 - pshufb xmm4,xmm0 - movdqa xmm2,xmm10 - pxor xmm4,xmm5 - pshufb xmm2,xmm3 - movdqa xmm3,xmm10 - pxor xmm2,xmm0 - pshufb xmm3,xmm4 +; top of round + movdqa xmm1,xmm9 ; 1 : i + movdqa xmm5,xmm11 ; 2 : a/k + pandn xmm1,xmm0 ; 1 = i<<4 + psrld xmm1,4 ; 1 = i + pand xmm0,xmm9 ; 0 = k + pshufb xmm5,xmm0 ; 2 = a/k + movdqa xmm3,xmm10 ; 3 : 1/i + pxor xmm0,xmm1 ; 0 = j + pshufb xmm3,xmm1 ; 3 = 1/i + movdqa xmm4,xmm10 ; 4 : 1/j + pxor xmm3,xmm5 ; 3 = iak = 1/i + a/k + pshufb xmm4,xmm0 ; 4 = 1/j + movdqa xmm2,xmm10 ; 2 : 1/iak + pxor xmm4,xmm5 ; 4 = jak = 1/j + a/k + pshufb xmm2,xmm3 ; 2 = 1/iak + movdqa xmm3,xmm10 ; 3 : 1/jak + pxor xmm2,xmm0 ; 2 = io + pshufb xmm3,xmm4 ; 3 = 1/jak movdqu xmm5,XMMWORD[r9] - pxor xmm3,xmm1 + pxor xmm3,xmm1 ; 3 = jo jnz NEAR $L$enc_loop - - movdqa xmm4,XMMWORD[((-96))+r10] - movdqa xmm0,XMMWORD[((-80))+r10] - pshufb xmm4,xmm2 - pxor xmm4,xmm5 - pshufb xmm0,xmm3 - movdqa xmm1,XMMWORD[64+r10*1+r11] - pxor xmm0,xmm4 +; middle of last round + movdqa xmm4,XMMWORD[((-96))+r10] ; 3 : sbou .Lk_sbo + movdqa xmm0,XMMWORD[((-80))+r10] ; 0 : sbot .Lk_sbo+16 + pshufb xmm4,xmm2 ; 4 = sbou + pxor xmm4,xmm5 ; 4 = sb1u + k + pshufb xmm0,xmm3 ; 0 = sb1t + movdqa xmm1,XMMWORD[64+r10*1+r11] ; .Lk_sr[] + pxor xmm0,xmm4 ; 0 = A pshufb xmm0,xmm1 ret - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +; # +; # _aes_encrypt_core_2x +; # +; # AES-encrypt %xmm0 and %xmm6 in parallel. +; # +; # Inputs: +; # %xmm0 and %xmm6 = input +; # %xmm9 and %xmm10 as in _vpaes_preheat +; # (%rdx) = scheduled keys +; # +; # Output in %xmm0 and %xmm6 +; # Clobbers %xmm1-%xmm5, %xmm7, %xmm8, %xmm11-%xmm13, %r9, %r10, %r11, %rax +; # Preserves %xmm14 and %xmm15 +; # +; # This function stitches two parallel instances of _vpaes_encrypt_core. x86_64 +; # provides 16 XMM registers. _vpaes_encrypt_core computes over six registers +; # (%xmm0-%xmm5) and additionally uses seven registers with preloaded constants +; # from _vpaes_preheat (%xmm9-%xmm15). This does not quite fit two instances, +; # so we spill some of %xmm9 through %xmm15 back to memory. We keep %xmm9 and +; # %xmm10 in registers as these values are used several times in a row. The +; # remainder are read once per round and are spilled to memory. This leaves two +; # registers preserved for the caller. +; # +; # Thus, of the two _vpaes_encrypt_core instances, the first uses (%xmm0-%xmm5) +; # as before. The second uses %xmm6-%xmm8,%xmm11-%xmm13. (Add 6 to %xmm2 and +; # below. Add 8 to %xmm3 and up.) Instructions in the second instance are +; # indented by one space. +; # +; # ALIGN 16 _vpaes_encrypt_core_2x: @@ -154,19 +154,19 @@ mov eax,DWORD[240+rdx] movdqa xmm1,xmm9 movdqa xmm7,xmm9 - movdqa xmm2,XMMWORD[$L$k_ipt] + movdqa xmm2,XMMWORD[$L$k_ipt] ; iptlo movdqa xmm8,xmm2 pandn xmm1,xmm0 pandn xmm7,xmm6 - movdqu xmm5,XMMWORD[r9] - + movdqu xmm5,XMMWORD[r9] ; round0 key +; Also use %xmm5 in the second instance. psrld xmm1,4 psrld xmm7,4 pand xmm0,xmm9 pand xmm6,xmm9 pshufb xmm2,xmm0 pshufb xmm8,xmm6 - movdqa xmm0,XMMWORD[(($L$k_ipt+16))] + movdqa xmm0,XMMWORD[(($L$k_ipt+16))] ; ipthi movdqa xmm6,xmm0 pshufb xmm0,xmm1 pshufb xmm6,xmm7 @@ -180,110 +180,110 @@ ALIGN 16 $L$enc2x_loop: - - movdqa xmm4,XMMWORD[$L$k_sb1] - movdqa xmm0,XMMWORD[(($L$k_sb1+16))] +; middle of middle round + movdqa xmm4,XMMWORD[$L$k_sb1] ; 4 : sb1u + movdqa xmm0,XMMWORD[(($L$k_sb1+16))] ; 0 : sb1t movdqa xmm12,xmm4 movdqa xmm6,xmm0 - pshufb xmm4,xmm2 + pshufb xmm4,xmm2 ; 4 = sb1u pshufb xmm12,xmm8 - pshufb xmm0,xmm3 + pshufb xmm0,xmm3 ; 0 = sb1t pshufb xmm6,xmm11 - pxor xmm4,xmm5 + pxor xmm4,xmm5 ; 4 = sb1u + k pxor xmm12,xmm5 - movdqa xmm5,XMMWORD[$L$k_sb2] + movdqa xmm5,XMMWORD[$L$k_sb2] ; 4 : sb2u movdqa xmm13,xmm5 - pxor xmm0,xmm4 + pxor xmm0,xmm4 ; 0 = A pxor xmm6,xmm12 - movdqa xmm1,XMMWORD[((-64))+r10*1+r11] - - pshufb xmm5,xmm2 + movdqa xmm1,XMMWORD[((-64))+r10*1+r11] ; .Lk_mc_forward[] +; Also use %xmm1 in the second instance. + pshufb xmm5,xmm2 ; 4 = sb2u pshufb xmm13,xmm8 - movdqa xmm4,XMMWORD[r10*1+r11] - - movdqa xmm2,XMMWORD[(($L$k_sb2+16))] + movdqa xmm4,XMMWORD[r10*1+r11] ; .Lk_mc_backward[] +; Also use %xmm4 in the second instance. + movdqa xmm2,XMMWORD[(($L$k_sb2+16))] ; 2 : sb2t movdqa xmm8,xmm2 - pshufb xmm2,xmm3 + pshufb xmm2,xmm3 ; 2 = sb2t pshufb xmm8,xmm11 - movdqa xmm3,xmm0 + movdqa xmm3,xmm0 ; 3 = A movdqa xmm11,xmm6 - pxor xmm2,xmm5 + pxor xmm2,xmm5 ; 2 = 2A pxor xmm8,xmm13 - pshufb xmm0,xmm1 + pshufb xmm0,xmm1 ; 0 = B pshufb xmm6,xmm1 - add r9,16 - pxor xmm0,xmm2 + add r9,16 ; next key + pxor xmm0,xmm2 ; 0 = 2A+B pxor xmm6,xmm8 - pshufb xmm3,xmm4 + pshufb xmm3,xmm4 ; 3 = D pshufb xmm11,xmm4 - add r11,16 - pxor xmm3,xmm0 + add r11,16 ; next mc + pxor xmm3,xmm0 ; 3 = 2A+B+D pxor xmm11,xmm6 - pshufb xmm0,xmm1 + pshufb xmm0,xmm1 ; 0 = 2B+C pshufb xmm6,xmm1 - and r11,0x30 - sub rax,1 - pxor xmm0,xmm3 + and r11,0x30 ; ... mod 4 + sub rax,1 ; nr-- + pxor xmm0,xmm3 ; 0 = 2A+3B+C+D pxor xmm6,xmm11 $L$enc2x_entry: - - movdqa xmm1,xmm9 +; top of round + movdqa xmm1,xmm9 ; 1 : i movdqa xmm7,xmm9 - movdqa xmm5,XMMWORD[(($L$k_inv+16))] + movdqa xmm5,XMMWORD[(($L$k_inv+16))] ; 2 : a/k movdqa xmm13,xmm5 - pandn xmm1,xmm0 + pandn xmm1,xmm0 ; 1 = i<<4 pandn xmm7,xmm6 - psrld xmm1,4 + psrld xmm1,4 ; 1 = i psrld xmm7,4 - pand xmm0,xmm9 + pand xmm0,xmm9 ; 0 = k pand xmm6,xmm9 - pshufb xmm5,xmm0 + pshufb xmm5,xmm0 ; 2 = a/k pshufb xmm13,xmm6 - movdqa xmm3,xmm10 + movdqa xmm3,xmm10 ; 3 : 1/i movdqa xmm11,xmm10 - pxor xmm0,xmm1 + pxor xmm0,xmm1 ; 0 = j pxor xmm6,xmm7 - pshufb xmm3,xmm1 + pshufb xmm3,xmm1 ; 3 = 1/i pshufb xmm11,xmm7 - movdqa xmm4,xmm10 + movdqa xmm4,xmm10 ; 4 : 1/j movdqa xmm12,xmm10 - pxor xmm3,xmm5 + pxor xmm3,xmm5 ; 3 = iak = 1/i + a/k pxor xmm11,xmm13 - pshufb xmm4,xmm0 + pshufb xmm4,xmm0 ; 4 = 1/j pshufb xmm12,xmm6 - movdqa xmm2,xmm10 + movdqa xmm2,xmm10 ; 2 : 1/iak movdqa xmm8,xmm10 - pxor xmm4,xmm5 + pxor xmm4,xmm5 ; 4 = jak = 1/j + a/k pxor xmm12,xmm13 - pshufb xmm2,xmm3 + pshufb xmm2,xmm3 ; 2 = 1/iak pshufb xmm8,xmm11 - movdqa xmm3,xmm10 + movdqa xmm3,xmm10 ; 3 : 1/jak movdqa xmm11,xmm10 - pxor xmm2,xmm0 + pxor xmm2,xmm0 ; 2 = io pxor xmm8,xmm6 - pshufb xmm3,xmm4 + pshufb xmm3,xmm4 ; 3 = 1/jak pshufb xmm11,xmm12 movdqu xmm5,XMMWORD[r9] - - pxor xmm3,xmm1 +; Also use %xmm5 in the second instance. + pxor xmm3,xmm1 ; 3 = jo pxor xmm11,xmm7 jnz NEAR $L$enc2x_loop - - movdqa xmm4,XMMWORD[((-96))+r10] - movdqa xmm0,XMMWORD[((-80))+r10] +; middle of last round + movdqa xmm4,XMMWORD[((-96))+r10] ; 3 : sbou .Lk_sbo + movdqa xmm0,XMMWORD[((-80))+r10] ; 0 : sbot .Lk_sbo+16 movdqa xmm12,xmm4 movdqa xmm6,xmm0 - pshufb xmm4,xmm2 + pshufb xmm4,xmm2 ; 4 = sbou pshufb xmm12,xmm8 - pxor xmm4,xmm5 + pxor xmm4,xmm5 ; 4 = sb1u + k pxor xmm12,xmm5 - pshufb xmm0,xmm3 + pshufb xmm0,xmm3 ; 0 = sb1t pshufb xmm6,xmm11 - movdqa xmm1,XMMWORD[64+r10*1+r11] - - pxor xmm0,xmm4 + movdqa xmm1,XMMWORD[64+r10*1+r11] ; .Lk_sr[] +; Also use %xmm1 in the second instance. + pxor xmm0,xmm4 ; 0 = A pxor xmm6,xmm12 pshufb xmm0,xmm1 pshufb xmm6,xmm1 @@ -291,27 +291,27 @@ - - - - - +; # +; # Decryption core +; # +; # Same API as encryption core. +; # ALIGN 16 _vpaes_decrypt_core: - mov r9,rdx + mov r9,rdx ; load key mov eax,DWORD[240+rdx] movdqa xmm1,xmm9 - movdqa xmm2,XMMWORD[$L$k_dipt] + movdqa xmm2,XMMWORD[$L$k_dipt] ; iptlo pandn xmm1,xmm0 mov r11,rax psrld xmm1,4 - movdqu xmm5,XMMWORD[r9] + movdqu xmm5,XMMWORD[r9] ; round0 key shl r11,4 pand xmm0,xmm9 pshufb xmm2,xmm0 - movdqa xmm0,XMMWORD[(($L$k_dipt+16))] + movdqa xmm0,XMMWORD[(($L$k_dipt+16))] ; ipthi xor r11,0x30 lea r10,[$L$k_dsbd] pshufb xmm0,xmm1 @@ -325,99 +325,99 @@ ALIGN 16 $L$dec_loop: - - - - movdqa xmm4,XMMWORD[((-32))+r10] - movdqa xmm1,XMMWORD[((-16))+r10] - pshufb xmm4,xmm2 - pshufb xmm1,xmm3 +; # +; # Inverse mix columns +; # + movdqa xmm4,XMMWORD[((-32))+r10] ; 4 : sb9u + movdqa xmm1,XMMWORD[((-16))+r10] ; 0 : sb9t + pshufb xmm4,xmm2 ; 4 = sb9u + pshufb xmm1,xmm3 ; 0 = sb9t pxor xmm0,xmm4 - movdqa xmm4,XMMWORD[r10] - pxor xmm0,xmm1 - movdqa xmm1,XMMWORD[16+r10] + movdqa xmm4,XMMWORD[r10] ; 4 : sbdu + pxor xmm0,xmm1 ; 0 = ch + movdqa xmm1,XMMWORD[16+r10] ; 0 : sbdt - pshufb xmm4,xmm2 - pshufb xmm0,xmm5 - pshufb xmm1,xmm3 - pxor xmm0,xmm4 - movdqa xmm4,XMMWORD[32+r10] - pxor xmm0,xmm1 - movdqa xmm1,XMMWORD[48+r10] + pshufb xmm4,xmm2 ; 4 = sbdu + pshufb xmm0,xmm5 ; MC ch + pshufb xmm1,xmm3 ; 0 = sbdt + pxor xmm0,xmm4 ; 4 = ch + movdqa xmm4,XMMWORD[32+r10] ; 4 : sbbu + pxor xmm0,xmm1 ; 0 = ch + movdqa xmm1,XMMWORD[48+r10] ; 0 : sbbt - pshufb xmm4,xmm2 - pshufb xmm0,xmm5 - pshufb xmm1,xmm3 - pxor xmm0,xmm4 - movdqa xmm4,XMMWORD[64+r10] - pxor xmm0,xmm1 - movdqa xmm1,XMMWORD[80+r10] + pshufb xmm4,xmm2 ; 4 = sbbu + pshufb xmm0,xmm5 ; MC ch + pshufb xmm1,xmm3 ; 0 = sbbt + pxor xmm0,xmm4 ; 4 = ch + movdqa xmm4,XMMWORD[64+r10] ; 4 : sbeu + pxor xmm0,xmm1 ; 0 = ch + movdqa xmm1,XMMWORD[80+r10] ; 0 : sbet - pshufb xmm4,xmm2 - pshufb xmm0,xmm5 - pshufb xmm1,xmm3 - pxor xmm0,xmm4 - add r9,16 + pshufb xmm4,xmm2 ; 4 = sbeu + pshufb xmm0,xmm5 ; MC ch + pshufb xmm1,xmm3 ; 0 = sbet + pxor xmm0,xmm4 ; 4 = ch + add r9,16 ; next round key palignr xmm5,xmm5,12 - pxor xmm0,xmm1 - sub rax,1 + pxor xmm0,xmm1 ; 0 = ch + sub rax,1 ; nr-- $L$dec_entry: - - movdqa xmm1,xmm9 - pandn xmm1,xmm0 - movdqa xmm2,xmm11 - psrld xmm1,4 - pand xmm0,xmm9 - pshufb xmm2,xmm0 - movdqa xmm3,xmm10 - pxor xmm0,xmm1 - pshufb xmm3,xmm1 - movdqa xmm4,xmm10 - pxor xmm3,xmm2 - pshufb xmm4,xmm0 - pxor xmm4,xmm2 - movdqa xmm2,xmm10 - pshufb xmm2,xmm3 - movdqa xmm3,xmm10 - pxor xmm2,xmm0 - pshufb xmm3,xmm4 +; top of round + movdqa xmm1,xmm9 ; 1 : i + pandn xmm1,xmm0 ; 1 = i<<4 + movdqa xmm2,xmm11 ; 2 : a/k + psrld xmm1,4 ; 1 = i + pand xmm0,xmm9 ; 0 = k + pshufb xmm2,xmm0 ; 2 = a/k + movdqa xmm3,xmm10 ; 3 : 1/i + pxor xmm0,xmm1 ; 0 = j + pshufb xmm3,xmm1 ; 3 = 1/i + movdqa xmm4,xmm10 ; 4 : 1/j + pxor xmm3,xmm2 ; 3 = iak = 1/i + a/k + pshufb xmm4,xmm0 ; 4 = 1/j + pxor xmm4,xmm2 ; 4 = jak = 1/j + a/k + movdqa xmm2,xmm10 ; 2 : 1/iak + pshufb xmm2,xmm3 ; 2 = 1/iak + movdqa xmm3,xmm10 ; 3 : 1/jak + pxor xmm2,xmm0 ; 2 = io + pshufb xmm3,xmm4 ; 3 = 1/jak movdqu xmm0,XMMWORD[r9] - pxor xmm3,xmm1 + pxor xmm3,xmm1 ; 3 = jo jnz NEAR $L$dec_loop - - movdqa xmm4,XMMWORD[96+r10] - pshufb xmm4,xmm2 - pxor xmm4,xmm0 - movdqa xmm0,XMMWORD[112+r10] - movdqa xmm2,XMMWORD[((-352))+r11] - pshufb xmm0,xmm3 - pxor xmm0,xmm4 +; middle of last round + movdqa xmm4,XMMWORD[96+r10] ; 3 : sbou + pshufb xmm4,xmm2 ; 4 = sbou + pxor xmm4,xmm0 ; 4 = sb1u + k + movdqa xmm0,XMMWORD[112+r10] ; 0 : sbot + movdqa xmm2,XMMWORD[((-352))+r11] ; .Lk_sr-.Lk_dsbd=-0x160 + pshufb xmm0,xmm3 ; 0 = sb1t + pxor xmm0,xmm4 ; 0 = A pshufb xmm0,xmm2 ret - - - - - +; ####################################################### +; # ## +; # AES key schedule ## +; # ## +; ####################################################### ALIGN 16 _vpaes_schedule_core: +; rdi = key +; rsi = size in bits +; rdx = buffer +; rcx = direction. 0=encrypt, 1=decrypt + call _vpaes_preheat ; load the tables + movdqa xmm8,XMMWORD[$L$k_rcon] ; load rcon + movdqu xmm0,XMMWORD[rdi] ; load key (unaligned) - - - - call _vpaes_preheat - movdqa xmm8,XMMWORD[$L$k_rcon] - movdqu xmm0,XMMWORD[rdi] - - +; input transform movdqa xmm3,xmm0 lea r11,[$L$k_ipt] call _vpaes_schedule_transform @@ -427,12 +427,12 @@ test rcx,rcx jnz NEAR $L$schedule_am_decrypting - +; encrypting, output zeroth round key after transform movdqu XMMWORD[rdx],xmm0 jmp NEAR $L$schedule_go $L$schedule_am_decrypting: - +; decrypting, output zeroth round key after shiftrows movdqa xmm1,XMMWORD[r10*1+r8] pshufb xmm3,xmm1 movdqu XMMWORD[rdx],xmm3 @@ -442,16 +442,16 @@ cmp esi,192 ja NEAR $L$schedule_256 je NEAR $L$schedule_192 +; 128: fall though - - - - - - - - - +; # +; # .schedule_128 +; # +; # 128-bit specific part of key schedule. +; # +; # This schedule is really simple, because all its parts +; # are accomplished by the subroutines. +; # $L$schedule_128: mov esi,10 @@ -459,73 +459,73 @@ call _vpaes_schedule_round dec rsi jz NEAR $L$schedule_mangle_last - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle ; write output jmp NEAR $L$oop_schedule_128 - - - - - - - - - - - - - - - +; # +; # .aes_schedule_192 +; # +; # 192-bit specific part of key schedule. +; # +; # The main body of this schedule is the same as the 128-bit +; # schedule, but with more smearing. The long, high side is +; # stored in %xmm7 as before, and the short, low side is in +; # the high bits of %xmm6. +; # +; # This schedule is somewhat nastier, however, because each +; # round produces 192 bits of key material, or 1.5 round keys. +; # Therefore, on each cycle we do 2 rounds and produce 3 round +; # keys. +; # ALIGN 16 $L$schedule_192: - movdqu xmm0,XMMWORD[8+rdi] - call _vpaes_schedule_transform - movdqa xmm6,xmm0 - pxor xmm4,xmm4 - movhlps xmm6,xmm4 + movdqu xmm0,XMMWORD[8+rdi] ; load key part 2 (very unaligned) + call _vpaes_schedule_transform ; input transform + movdqa xmm6,xmm0 ; save short part + pxor xmm4,xmm4 ; clear 4 + movhlps xmm6,xmm4 ; clobber low side with zeros mov esi,4 $L$oop_schedule_192: call _vpaes_schedule_round palignr xmm0,xmm6,8 - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle ; save key n call _vpaes_schedule_192_smear - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle ; save key n+1 call _vpaes_schedule_round dec rsi jz NEAR $L$schedule_mangle_last - call _vpaes_schedule_mangle + call _vpaes_schedule_mangle ; save key n+2 call _vpaes_schedule_192_smear jmp NEAR $L$oop_schedule_192 - - - - - - - - - - +; # +; # .aes_schedule_256 +; # +; # 256-bit specific part of key schedule. +; # +; # The structure here is very similar to the 128-bit +; # schedule, but with an additional "low side" in +; # %xmm6. The low side's rounds are the same as the +; # high side's, except no rcon and no rotation. +; # ALIGN 16 $L$schedule_256: - movdqu xmm0,XMMWORD[16+rdi] - call _vpaes_schedule_transform + movdqu xmm0,XMMWORD[16+rdi] ; load key part 2 (unaligned) + call _vpaes_schedule_transform ; input transform mov esi,7 $L$oop_schedule_256: - call _vpaes_schedule_mangle - movdqa xmm6,xmm0 + call _vpaes_schedule_mangle ; output low result + movdqa xmm6,xmm0 ; save cur_lo in xmm6 - +; high round call _vpaes_schedule_round dec rsi jz NEAR $L$schedule_mangle_last call _vpaes_schedule_mangle - +; low round. swap xmm7 and xmm6 pshufd xmm0,xmm0,0xFF movdqa xmm5,xmm7 movdqa xmm7,xmm6 @@ -535,36 +535,36 @@ jmp NEAR $L$oop_schedule_256 - - - - - - - - - - +; # +; # .aes_schedule_mangle_last +; # +; # Mangler for last round of key schedule +; # Mangles %xmm0 +; # when encrypting, outputs out(%xmm0) ^ 63 +; # when decrypting, outputs unskew(%xmm0) +; # +; # Always called right before return... jumps to cleanup and exits +; # ALIGN 16 $L$schedule_mangle_last: - - lea r11,[$L$k_deskew] +; schedule last round key from xmm0 + lea r11,[$L$k_deskew] ; prepare to deskew test rcx,rcx jnz NEAR $L$schedule_mangle_last_dec - +; encrypting movdqa xmm1,XMMWORD[r10*1+r8] - pshufb xmm0,xmm1 - lea r11,[$L$k_opt] + pshufb xmm0,xmm1 ; output permute + lea r11,[$L$k_opt] ; prepare to output transform add rdx,32 $L$schedule_mangle_last_dec: add rdx,-16 pxor xmm0,XMMWORD[$L$k_s63] - call _vpaes_schedule_transform - movdqu XMMWORD[rdx],xmm0 + call _vpaes_schedule_transform ; output transform + movdqu XMMWORD[rdx],xmm0 ; save last key - +; cleanup pxor xmm0,xmm0 pxor xmm1,xmm1 pxor xmm2,xmm2 @@ -577,72 +577,72 @@ - - - - - - - - - - - - - - +; # +; # .aes_schedule_192_smear +; # +; # Smear the short, low side in the 192-bit key schedule. +; # +; # Inputs: +; # %xmm7: high side, b a x y +; # %xmm6: low side, d c 0 0 +; # %xmm13: 0 +; # +; # Outputs: +; # %xmm6: b+c+d b+c 0 0 +; # %xmm0: b+c+d b+c b a +; # ALIGN 16 _vpaes_schedule_192_smear: - pshufd xmm1,xmm6,0x80 - pshufd xmm0,xmm7,0xFE - pxor xmm6,xmm1 + pshufd xmm1,xmm6,0x80 ; d c 0 0 -> c 0 0 0 + pshufd xmm0,xmm7,0xFE ; b a _ _ -> b b b a + pxor xmm6,xmm1 ; -> c+d c 0 0 pxor xmm1,xmm1 - pxor xmm6,xmm0 + pxor xmm6,xmm0 ; -> b+c+d b+c b a movdqa xmm0,xmm6 - movhlps xmm6,xmm1 + movhlps xmm6,xmm1 ; clobber low side with zeros ret - - - - - - - - - - - - - - - - - - +; # +; # .aes_schedule_round +; # +; # Runs one main round of the key schedule on %xmm0, %xmm7 +; # +; # Specifically, runs subbytes on the high dword of %xmm0 +; # then rotates it by one byte and xors into the low dword of +; # %xmm7. +; # +; # Adds rcon from low byte of %xmm8, then rotates %xmm8 for +; # next rcon. +; # +; # Smears the dwords of %xmm7 by xoring the low into the +; # second low, result into third, result into highest. +; # +; # Returns results in %xmm7 = %xmm0. +; # Clobbers %xmm1-%xmm4, %r11. +; # ALIGN 16 _vpaes_schedule_round: - +; extract rcon from xmm8 pxor xmm1,xmm1 palignr xmm1,xmm8,15 palignr xmm8,xmm8,15 pxor xmm7,xmm1 - +; rotate pshufd xmm0,xmm0,0xFF palignr xmm0,xmm0,1 +; fall through... - - +; low round: same as high round, but no rotation and no rcon. _vpaes_schedule_low_round: - +; smear xmm7 movdqa xmm1,xmm7 pslldq xmm7,4 pxor xmm7,xmm1 @@ -651,48 +651,48 @@ pxor xmm7,xmm1 pxor xmm7,XMMWORD[$L$k_s63] - +; subbytes movdqa xmm1,xmm9 pandn xmm1,xmm0 - psrld xmm1,4 - pand xmm0,xmm9 - movdqa xmm2,xmm11 - pshufb xmm2,xmm0 - pxor xmm0,xmm1 - movdqa xmm3,xmm10 - pshufb xmm3,xmm1 - pxor xmm3,xmm2 - movdqa xmm4,xmm10 - pshufb xmm4,xmm0 - pxor xmm4,xmm2 - movdqa xmm2,xmm10 - pshufb xmm2,xmm3 - pxor xmm2,xmm0 - movdqa xmm3,xmm10 - pshufb xmm3,xmm4 - pxor xmm3,xmm1 - movdqa xmm4,xmm13 - pshufb xmm4,xmm2 - movdqa xmm0,xmm12 - pshufb xmm0,xmm3 - pxor xmm0,xmm4 + psrld xmm1,4 ; 1 = i + pand xmm0,xmm9 ; 0 = k + movdqa xmm2,xmm11 ; 2 : a/k + pshufb xmm2,xmm0 ; 2 = a/k + pxor xmm0,xmm1 ; 0 = j + movdqa xmm3,xmm10 ; 3 : 1/i + pshufb xmm3,xmm1 ; 3 = 1/i + pxor xmm3,xmm2 ; 3 = iak = 1/i + a/k + movdqa xmm4,xmm10 ; 4 : 1/j + pshufb xmm4,xmm0 ; 4 = 1/j + pxor xmm4,xmm2 ; 4 = jak = 1/j + a/k + movdqa xmm2,xmm10 ; 2 : 1/iak + pshufb xmm2,xmm3 ; 2 = 1/iak + pxor xmm2,xmm0 ; 2 = io + movdqa xmm3,xmm10 ; 3 : 1/jak + pshufb xmm3,xmm4 ; 3 = 1/jak + pxor xmm3,xmm1 ; 3 = jo + movdqa xmm4,xmm13 ; 4 : sbou + pshufb xmm4,xmm2 ; 4 = sbou + movdqa xmm0,xmm12 ; 0 : sbot + pshufb xmm0,xmm3 ; 0 = sb1t + pxor xmm0,xmm4 ; 0 = sbox output - +; add in smeared stuff pxor xmm0,xmm7 movdqa xmm7,xmm0 ret - - - - - - - - - +; # +; # .aes_schedule_transform +; # +; # Linear-transform %xmm0 according to tables at (%r11) +; # +; # Requires that %xmm9 = 0x0F0F... as in preheat +; # Output in %xmm0 +; # Clobbers %xmm1, %xmm2 +; # ALIGN 16 _vpaes_schedule_transform: @@ -701,48 +701,48 @@ pandn xmm1,xmm0 psrld xmm1,4 pand xmm0,xmm9 - movdqa xmm2,XMMWORD[r11] + movdqa xmm2,XMMWORD[r11] ; lo pshufb xmm2,xmm0 - movdqa xmm0,XMMWORD[16+r11] + movdqa xmm0,XMMWORD[16+r11] ; hi pshufb xmm0,xmm1 pxor xmm0,xmm2 ret - - - - - - - - - - - - - - - - - - - - - - - +; # +; # .aes_schedule_mangle +; # +; # Mangle xmm0 from (basis-transformed) standard version +; # to our version. +; # +; # On encrypt, +; # xor with 0x63 +; # multiply by circulant 0,1,1,1 +; # apply shiftrows transform +; # +; # On decrypt, +; # xor with 0x63 +; # multiply by "inverse mixcolumns" circulant E,B,D,9 +; # deskew +; # apply shiftrows transform +; # +; # +; # Writes out to (%rdx), and increments or decrements it +; # Keeps track of round number mod 4 in %r8 +; # Preserves xmm0 +; # Clobbers xmm1-xmm5 +; # ALIGN 16 _vpaes_schedule_mangle: - movdqa xmm4,xmm0 + movdqa xmm4,xmm0 ; save xmm0 for later movdqa xmm5,XMMWORD[$L$k_mc_forward] test rcx,rcx jnz NEAR $L$schedule_mangle_dec - +; encrypting add rdx,16 pxor xmm4,XMMWORD[$L$k_s63] pshufb xmm4,xmm5 @@ -755,12 +755,12 @@ jmp NEAR $L$schedule_mangle_both ALIGN 16 $L$schedule_mangle_dec: - +; inverse mix columns lea r11,[$L$k_dksd] movdqa xmm1,xmm9 pandn xmm1,xmm4 - psrld xmm1,4 - pand xmm4,xmm9 + psrld xmm1,4 ; 1 = hi + pand xmm4,xmm9 ; 4 = lo movdqa xmm2,XMMWORD[r11] pshufb xmm2,xmm4 @@ -804,9 +804,9 @@ - - - +; +; Interface to OpenSSL +; global vpaes_set_encrypt_key ALIGN 16 @@ -842,7 +842,7 @@ mov eax,esi shr eax,5 add eax,5 - mov DWORD[240+rdx],eax + mov DWORD[240+rdx],eax ; AES_KEY->rounds = nbits/32+5; mov ecx,0 mov r8d,0x30 @@ -896,7 +896,7 @@ mov eax,esi shr eax,5 add eax,5 - mov DWORD[240+rdx],eax + mov DWORD[240+rdx],eax ; AES_KEY->rounds = nbits/32+5; shl eax,4 lea rdx,[16+rax*1+rdx] @@ -904,7 +904,7 @@ mov r8d,esi shr r8d,1 and r8d,32 - xor r8d,32 + xor r8d,32 ; nbits==192?0:32 call _vpaes_schedule_core movaps xmm6,XMMWORD[16+rsp] movaps xmm7,XMMWORD[32+rsp] @@ -1059,7 +1059,7 @@ movaps XMMWORD[144+rsp],xmm14 movaps XMMWORD[160+rsp],xmm15 $L$cbc_body: - movdqu xmm6,XMMWORD[r8] + movdqu xmm6,XMMWORD[r8] ; load IV sub rsi,rdi call _vpaes_preheat cmp r9d,0 @@ -1088,7 +1088,7 @@ sub rcx,16 jnc NEAR $L$cbc_dec_loop $L$cbc_done: - movdqu XMMWORD[r8],xmm6 + movdqu XMMWORD[r8],xmm6 ; save IV movaps xmm6,XMMWORD[16+rsp] movaps xmm7,XMMWORD[32+rsp] movaps xmm8,XMMWORD[48+rsp] @@ -1124,7 +1124,7 @@ _CET_ENDBR - +; _vpaes_encrypt_core and _vpaes_encrypt_core_2x expect the key in %rdx. xchg rdx,rcx test rcx,rcx jz NEAR $L$ctr32_abort @@ -1140,9 +1140,9 @@ movaps XMMWORD[144+rsp],xmm14 movaps XMMWORD[160+rsp],xmm15 $L$ctr32_body: - movdqu xmm0,XMMWORD[r8] + movdqu xmm0,XMMWORD[r8] ; Load IV. movdqa xmm8,XMMWORD[$L$ctr_add_one] - sub rsi,rdi + sub rsi,rdi ; This allows only incrementing %rdi. call _vpaes_preheat movdqa xmm6,xmm0 pshufb xmm6,XMMWORD[$L$rev_ctr] @@ -1150,9 +1150,9 @@ test rcx,1 jz NEAR $L$ctr32_prep_loop - - - movdqu xmm7,XMMWORD[rdi] +; Handle one block so the remaining block count is even for +; _vpaes_encrypt_core_2x. + movdqu xmm7,XMMWORD[rdi] ; Load input. call _vpaes_encrypt_core pxor xmm0,xmm7 paddd xmm6,xmm8 @@ -1162,29 +1162,29 @@ jz NEAR $L$ctr32_done $L$ctr32_prep_loop: - - +; _vpaes_encrypt_core_2x leaves only %xmm14 and %xmm15 as spare +; registers. We maintain two byte-swapped counters in them. movdqa xmm14,xmm6 movdqa xmm15,xmm6 paddd xmm15,xmm8 $L$ctr32_loop: - movdqa xmm1,XMMWORD[$L$rev_ctr] + movdqa xmm1,XMMWORD[$L$rev_ctr] ; Set up counters. movdqa xmm0,xmm14 movdqa xmm6,xmm15 pshufb xmm0,xmm1 pshufb xmm6,xmm1 call _vpaes_encrypt_core_2x - movdqu xmm1,XMMWORD[rdi] + movdqu xmm1,XMMWORD[rdi] ; Load input. movdqu xmm2,XMMWORD[16+rdi] movdqa xmm3,XMMWORD[$L$ctr_add_two] - pxor xmm0,xmm1 + pxor xmm0,xmm1 ; XOR input. pxor xmm6,xmm2 - paddd xmm14,xmm3 + paddd xmm14,xmm3 ; Increment counters. paddd xmm15,xmm3 - movdqu XMMWORD[rdi*1+rsi],xmm0 + movdqu XMMWORD[rdi*1+rsi],xmm0 ; Write output. movdqu XMMWORD[16+rdi*1+rsi],xmm6 - sub rcx,2 + sub rcx,2 ; Advance loop. lea rdi,[32+rdi] jnz NEAR $L$ctr32_loop @@ -1207,135 +1207,135 @@ ret $L$SEH_end_vpaes_ctr32_encrypt_blocks: - - - - - - +; # +; # _aes_preheat +; # +; # Fills register %r10 -> .aes_consts (so you can -fPIC) +; # and %xmm9-%xmm15 as specified below. +; # ALIGN 16 _vpaes_preheat: lea r10,[$L$k_s0F] - movdqa xmm10,XMMWORD[((-32))+r10] - movdqa xmm11,XMMWORD[((-16))+r10] - movdqa xmm9,XMMWORD[r10] - movdqa xmm13,XMMWORD[48+r10] - movdqa xmm12,XMMWORD[64+r10] - movdqa xmm15,XMMWORD[80+r10] - movdqa xmm14,XMMWORD[96+r10] + movdqa xmm10,XMMWORD[((-32))+r10] ; .Lk_inv + movdqa xmm11,XMMWORD[((-16))+r10] ; .Lk_inv+16 + movdqa xmm9,XMMWORD[r10] ; .Lk_s0F + movdqa xmm13,XMMWORD[48+r10] ; .Lk_sb1 + movdqa xmm12,XMMWORD[64+r10] ; .Lk_sb1+16 + movdqa xmm15,XMMWORD[80+r10] ; .Lk_sb2 + movdqa xmm14,XMMWORD[96+r10] ; .Lk_sb2+16 ret - - - - - +; ####################################################### +; # ## +; # Constants ## +; # ## +; ####################################################### section .rdata rdata align=8 ALIGN 64 _vpaes_consts: -$L$k_inv: +$L$k_inv: ; inv, inva DQ 0x0E05060F0D080180,0x040703090A0B0C02 DQ 0x01040A060F0B0780,0x030D0E0C02050809 -$L$k_s0F: +$L$k_s0F: ; s0F DQ 0x0F0F0F0F0F0F0F0F,0x0F0F0F0F0F0F0F0F -$L$k_ipt: +$L$k_ipt: ; input transform (lo, hi) DQ 0xC2B2E8985A2A7000,0xCABAE09052227808 DQ 0x4C01307D317C4D00,0xCD80B1FCB0FDCC81 -$L$k_sb1: +$L$k_sb1: ; sb1u, sb1t DQ 0xB19BE18FCB503E00,0xA5DF7A6E142AF544 DQ 0x3618D415FAE22300,0x3BF7CCC10D2ED9EF -$L$k_sb2: +$L$k_sb2: ; sb2u, sb2t DQ 0xE27A93C60B712400,0x5EB7E955BC982FCD DQ 0x69EB88400AE12900,0xC2A163C8AB82234A -$L$k_sbo: +$L$k_sbo: ; sbou, sbot DQ 0xD0D26D176FBDC700,0x15AABF7AC502A878 DQ 0xCFE474A55FBB6A00,0x8E1E90D1412B35FA -$L$k_mc_forward: +$L$k_mc_forward: ; mc_forward DQ 0x0407060500030201,0x0C0F0E0D080B0A09 DQ 0x080B0A0904070605,0x000302010C0F0E0D DQ 0x0C0F0E0D080B0A09,0x0407060500030201 DQ 0x000302010C0F0E0D,0x080B0A0904070605 -$L$k_mc_backward: +$L$k_mc_backward: ; mc_backward DQ 0x0605040702010003,0x0E0D0C0F0A09080B DQ 0x020100030E0D0C0F,0x0A09080B06050407 DQ 0x0E0D0C0F0A09080B,0x0605040702010003 DQ 0x0A09080B06050407,0x020100030E0D0C0F -$L$k_sr: +$L$k_sr: ; sr DQ 0x0706050403020100,0x0F0E0D0C0B0A0908 DQ 0x030E09040F0A0500,0x0B06010C07020D08 DQ 0x0F060D040B020900,0x070E050C030A0108 DQ 0x0B0E0104070A0D00,0x0306090C0F020508 -$L$k_rcon: +$L$k_rcon: ; rcon DQ 0x1F8391B9AF9DEEB6,0x702A98084D7C7D81 -$L$k_s63: +$L$k_s63: ; s63: all equal to 0x63 transformed DQ 0x5B5B5B5B5B5B5B5B,0x5B5B5B5B5B5B5B5B -$L$k_opt: +$L$k_opt: ; output transform DQ 0xFF9F4929D6B66000,0xF7974121DEBE6808 DQ 0x01EDBD5150BCEC00,0xE10D5DB1B05C0CE0 -$L$k_deskew: +$L$k_deskew: ; deskew tables: inverts the sbox's "skew" DQ 0x07E4A34047A4E300,0x1DFEB95A5DBEF91A DQ 0x5F36B5DC83EA6900,0x2841C2ABF49D1E77 - - - - -$L$k_dksd: +; # +; # Decryption stuff +; # Key schedule constants +; # +$L$k_dksd: ; decryption key schedule: invskew x*D DQ 0xFEB91A5DA3E44700,0x0740E3A45A1DBEF9 DQ 0x41C277F4B5368300,0x5FDC69EAAB289D1E -$L$k_dksb: +$L$k_dksb: ; decryption key schedule: invskew x*B DQ 0x9A4FCA1F8550D500,0x03D653861CC94C99 DQ 0x115BEDA7B6FC4A00,0xD993256F7E3482C8 -$L$k_dkse: +$L$k_dkse: ; decryption key schedule: invskew x*E + 0x63 DQ 0xD5031CCA1FC9D600,0x53859A4C994F5086 DQ 0xA23196054FDC7BE8,0xCD5EF96A20B31487 -$L$k_dks9: +$L$k_dks9: ; decryption key schedule: invskew x*9 DQ 0xB6116FC87ED9A700,0x4AED933482255BFC DQ 0x4576516227143300,0x8BB89FACE9DAFDCE - - - - -$L$k_dipt: +; # +; # Decryption stuff +; # Round function constants +; # +$L$k_dipt: ; decryption input transform DQ 0x0F505B040B545F00,0x154A411E114E451A DQ 0x86E383E660056500,0x12771772F491F194 -$L$k_dsb9: +$L$k_dsb9: ; decryption sbox output *9*u, *9*t DQ 0x851C03539A86D600,0xCAD51F504F994CC9 DQ 0xC03B1789ECD74900,0x725E2C9EB2FBA565 -$L$k_dsbd: +$L$k_dsbd: ; decryption sbox output *D*u, *D*t DQ 0x7D57CCDFE6B1A200,0xF56E9B13882A4439 DQ 0x3CE2FAF724C6CB00,0x2931180D15DEEFD3 -$L$k_dsbb: +$L$k_dsbb: ; decryption sbox output *B*u, *B*t DQ 0xD022649296B44200,0x602646F6B0F2D404 DQ 0xC19498A6CD596700,0xF3FF0C3E3255AA6B -$L$k_dsbe: +$L$k_dsbe: ; decryption sbox output *E*u, *E*t DQ 0x46F2929626D4D000,0x2242600464B4F6B0 DQ 0x0C55A6CDFFAAC100,0x9467F36B98593E32 -$L$k_dsbo: +$L$k_dsbo: ; decryption sbox final output DQ 0x1387EA537EF94000,0xC7AA6DB9D4943E2D DQ 0x12D7560F93441D00,0xCA4B8159D8C58E9C - +; .Lrev_ctr is a permutation which byte-swaps the counter portion of the IV. $L$rev_ctr: DQ 0x0706050403020100,0x0c0d0e0f0b0a0908 - - +; .Lctr_add_* may be added to a byte-swapped xmm register to increment the +; counter. The register must be byte-swapped again to form the actual input. $L$ctr_add_one: DQ 0x0000000000000000,0x0000000100000000 $L$ctr_add_two: @@ -1365,57 +1365,57 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$in_prologue - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$in_prologue - lea rsi,[16+rax] - lea rdi,[512+r8] - mov ecx,20 - DD 0xa548f3fc - lea rax,[184+rax] + lea rsi,[16+rax] ; %xmm save area + lea rdi,[512+r8] ; &context.Xmm6 + mov ecx,20 ; 10*sizeof(%xmm0)/sizeof(%rax) + DD 0xa548f3fc ; cld; rep movsq + lea rax,[184+rax] ; adjust stack pointer $L$in_prologue: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -1460,27 +1460,27 @@ $L$SEH_info_vpaes_set_encrypt_key: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$enc_key_body wrt ..imagebase,$L$enc_key_epilogue wrt ..imagebase + DD $L$enc_key_body wrt ..imagebase,$L$enc_key_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_vpaes_set_decrypt_key: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$dec_key_body wrt ..imagebase,$L$dec_key_epilogue wrt ..imagebase + DD $L$dec_key_body wrt ..imagebase,$L$dec_key_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_vpaes_encrypt: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$enc_body wrt ..imagebase,$L$enc_epilogue wrt ..imagebase + DD $L$enc_body wrt ..imagebase,$L$enc_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_vpaes_decrypt: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$dec_body wrt ..imagebase,$L$dec_epilogue wrt ..imagebase + DD $L$dec_body wrt ..imagebase,$L$dec_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_vpaes_cbc_encrypt: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$cbc_body wrt ..imagebase,$L$cbc_epilogue wrt ..imagebase + DD $L$cbc_body wrt ..imagebase,$L$cbc_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_vpaes_ctr32_encrypt_blocks: DB 9,0,0,0 DD se_handler wrt ..imagebase - DD $L$ctr32_body wrt ..imagebase,$L$ctr32_epilogue wrt ..imagebase + DD $L$ctr32_body wrt ..imagebase,$L$ctr32_epilogue wrt ..imagebase ; HandlerData[] %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/bcm/x86_64-mont-apple.S b/gen/bcm/x86_64-mont-apple.S index 716ce60..714cd12 100644 --- a/gen/bcm/x86_64-mont-apple.S +++ b/gen/bcm/x86_64-mont-apple.S
@@ -31,18 +31,18 @@ negq %r9 movq %rsp,%r11 - leaq -16(%rsp,%r9,8),%r10 - negq %r9 - andq $-1024,%r10 + leaq -16(%rsp,%r9,8),%r10 // future alloca(8*(num+2)) + negq %r9 // restore %r9 + andq $-1024,%r10 // minimize TLB usage - - - - - - - - +// An OS-agnostic version of __chkstk. +// +// Some OSes (Windows) insist on stack being "wired" to +// physical memory in strictly sequential manner, i.e. if stack +// allocation spans two pages, then reference to farmost one can +// be punishable by SEGV. But page walking can do good even on +// other OSes, because it guarantees that villain thread hits +// the guard page before it can make damage to innocent one... subq %r10,%r11 andq $-4096,%r11 leaq (%r10,%r11,1),%rsp @@ -59,32 +59,32 @@ ja L$mul_page_walk L$mul_page_walk_done: - movq %rax,8(%rsp,%r9,8) + movq %rax,8(%rsp,%r9,8) // tp[num+1]=%rsp L$mul_body: - movq %rdx,%r12 - movq (%r8),%r8 - movq (%r12),%rbx + movq %rdx,%r12 // reassign %rdx + movq (%r8),%r8 // pull n0[0] value + movq (%r12),%rbx // m0=bp[0] movq (%rsi),%rax - xorq %r14,%r14 - xorq %r15,%r15 + xorq %r14,%r14 // i=0 + xorq %r15,%r15 // j=0 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp L$1st_enter .p2align 4 @@ -92,30 +92,30 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] movq %r10,%r11 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 L$1st_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 jne L$1st addq %rax,%r13 - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 movq %r10,%r11 @@ -123,32 +123,32 @@ addq %r11,%r13 adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ jmp L$outer .p2align 4 L$outer: - movq (%r12,%r14,8),%rbx - xorq %r15,%r15 + movq (%r12,%r14,8),%rbx // m0=bp[i] + xorq %r15,%r15 // j=0 movq %r8,%rbp movq (%rsp),%r10 - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx - movq 8(%rsp),%r10 + movq 8(%rsp),%r10 // tp[1] movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp L$inner_enter .p2align 4 @@ -156,80 +156,80 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r15,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 L$inner_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - addq %r11,%r10 + addq %r11,%r10 // ap[j]*bp[i]+tp[j] movq %rdx,%r11 adcq $0,%r11 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 jne L$inner addq %rax,%r13 - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r15,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 xorq %rdx,%rdx addq %r11,%r13 adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // pull upmost overflow bit adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ cmpq %r9,%r14 jb L$outer - xorq %r14,%r14 - movq (%rsp),%rax - movq %r9,%r15 + xorq %r14,%r14 // i=0 and clear CF + movq (%rsp),%rax // tp[0] + movq %r9,%r15 // j=num .p2align 4 L$sub: sbbq (%rcx,%r14,8),%rax - movq %rax,(%rdi,%r14,8) - movq 8(%rsp,%r14,8),%rax - leaq 1(%r14),%r14 - decq %r15 + movq %rax,(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq 8(%rsp,%r14,8),%rax // tp[i+1] + leaq 1(%r14),%r14 // i++ + decq %r15 // doesn't affect CF jnz L$sub - sbbq $0,%rax + sbbq $0,%rax // handle upmost overflow bit movq $-1,%rbx - xorq %rax,%rbx + xorq %rax,%rbx // not %rax xorq %r14,%r14 - movq %r9,%r15 + movq %r9,%r15 // j=num -L$copy: +L$copy: // conditional copy movq (%rdi,%r14,8),%rcx movq (%rsp,%r14,8),%rdx andq %rbx,%rcx andq %rax,%rdx - movq %r9,(%rsp,%r14,8) + movq %r9,(%rsp,%r14,8) // zap temporary vector orq %rcx,%rdx - movq %rdx,(%rdi,%r14,8) + movq %rdx,(%rdi,%r14,8) // rp[i]=tp[i] leaq 1(%r14),%r14 subq $1,%r15 jnz L$copy - movq 8(%rsp,%r9,8),%rsi + movq 8(%rsp,%r9,8),%rsi // restore %rsp - +// No return value movq -48(%rsi),%r15 movq -40(%rsi),%r14 @@ -274,9 +274,9 @@ negq %r9 movq %rsp,%r11 - leaq -32(%rsp,%r9,8),%r10 - negq %r9 - andq $-1024,%r10 + leaq -32(%rsp,%r9,8),%r10 // future alloca(8*(num+4)) + negq %r9 // restore + andq $-1024,%r10 // minimize TLB usage subq %r10,%r11 andq $-4096,%r11 @@ -293,28 +293,28 @@ ja L$mul4x_page_walk L$mul4x_page_walk_done: - movq %rax,8(%rsp,%r9,8) + movq %rax,8(%rsp,%r9,8) // tp[num+1]=%rsp L$mul4x_body: - movq %rdi,16(%rsp,%r9,8) - movq %rdx,%r12 - movq (%r8),%r8 - movq (%r12),%rbx + movq %rdi,16(%rsp,%r9,8) // tp[num+2]=%rdi + movq %rdx,%r12 // reassign %r12 + movq (%r8),%r8 // pull n0[0] value + movq (%r12),%rbx // m0=bp[0] movq (%rsi),%rax - xorq %r14,%r14 - xorq %r15,%r15 + xorq %r14,%r14 // i=0 + xorq %r15,%r15 // j=0 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%rdi @@ -330,171 +330,171 @@ movq 16(%rsi),%rax adcq $0,%rdx addq %r11,%rdi - leaq 4(%r15),%r15 + leaq 4(%r15),%r15 // j++ adcq $0,%rdx movq %rdi,(%rsp) movq %rdx,%r13 jmp L$1st4x .p2align 4 L$1st4x: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq (%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-8(%rsp,%r15,8) + movq %r13,-8(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq 8(%rcx,%r15,8),%rax adcq $0,%rdx - leaq 4(%r15),%r15 + leaq 4(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq -16(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-32(%rsp,%r15,8) + movq %rdi,-32(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 cmpq %r9,%r15 jb L$1st4x - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 xorq %rdi,%rdi addq %r10,%r13 adcq $0,%rdi movq %r13,-8(%rsp,%r15,8) - movq %rdi,(%rsp,%r15,8) + movq %rdi,(%rsp,%r15,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ .p2align 2 L$outer4x: - movq (%r12,%r14,8),%rbx - xorq %r15,%r15 + movq (%r12,%r14,8),%rbx // m0=bp[i] + xorq %r15,%r15 // j=0 movq (%rsp),%r10 movq %r8,%rbp - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // "%r13", discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx - addq 8(%rsp),%r11 + addq 8(%rsp),%r11 // +tp[1] adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi),%rax adcq $0,%rdx - addq %r11,%rdi - leaq 4(%r15),%r15 + addq %r11,%rdi // np[j]*m1+ap[j]*bp[i]+tp[j] + leaq 4(%r15),%r15 // j+=2 adcq $0,%rdx - movq %rdi,(%rsp) + movq %rdi,(%rsp) // tp[j-1] movq %rdx,%r13 jmp L$inner4x .p2align 4 L$inner4x: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx - addq -16(%rsp,%r15,8),%r10 + addq -16(%rsp,%r15,8),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx @@ -502,148 +502,148 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,8),%rax adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq (%rcx,%r15,8),%rax adcq $0,%rdx - addq (%rsp,%r15,8),%r10 + addq (%rsp,%r15,8),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,8),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %r13,-8(%rsp,%r15,8) + movq %r13,-8(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx,%r15,8),%rax adcq $0,%rdx addq 8(%rsp,%r15,8),%r11 adcq $0,%rdx - leaq 4(%r15),%r15 + leaq 4(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq -16(%rsi,%r15,8),%rax adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %rdi,-32(%rsp,%r15,8) + movq %rdi,-32(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 cmpq %r9,%r15 jb L$inner4x - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx - addq -16(%rsp,%r15,8),%r10 + addq -16(%rsp,%r15,8),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx addq -8(%rsp,%r15,8),%r11 adcq $0,%rdx - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 xorq %rdi,%rdi addq %r10,%r13 adcq $0,%rdi - addq (%rsp,%r9,8),%r13 + addq (%rsp,%r9,8),%r13 // pull upmost overflow bit adcq $0,%rdi movq %r13,-8(%rsp,%r15,8) - movq %rdi,(%rsp,%r15,8) + movq %rdi,(%rsp,%r15,8) // store upmost overflow bit cmpq %r9,%r14 jb L$outer4x - movq 16(%rsp,%r9,8),%rdi + movq 16(%rsp,%r9,8),%rdi // restore %rdi leaq -4(%r9),%r15 - movq 0(%rsp),%rax - movq 8(%rsp),%rdx - shrq $2,%r15 - leaq (%rsp),%rsi - xorq %r14,%r14 + movq 0(%rsp),%rax // tp[0] + movq 8(%rsp),%rdx // tp[1] + shrq $2,%r15 // j=num/4-1 + leaq (%rsp),%rsi // borrow ap for tp + xorq %r14,%r14 // i=0 and clear CF subq 0(%rcx),%rax - movq 16(%rsi),%rbx - movq 24(%rsi),%rbp + movq 16(%rsi),%rbx // tp[2] + movq 24(%rsi),%rbp // tp[3] sbbq 8(%rcx),%rdx L$sub4x: - movq %rax,0(%rdi,%r14,8) - movq %rdx,8(%rdi,%r14,8) + movq %rax,0(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq %rdx,8(%rdi,%r14,8) // rp[i]=tp[i]-np[i] sbbq 16(%rcx,%r14,8),%rbx - movq 32(%rsi,%r14,8),%rax + movq 32(%rsi,%r14,8),%rax // tp[i+1] movq 40(%rsi,%r14,8),%rdx sbbq 24(%rcx,%r14,8),%rbp - movq %rbx,16(%rdi,%r14,8) - movq %rbp,24(%rdi,%r14,8) + movq %rbx,16(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq %rbp,24(%rdi,%r14,8) // rp[i]=tp[i]-np[i] sbbq 32(%rcx,%r14,8),%rax movq 48(%rsi,%r14,8),%rbx movq 56(%rsi,%r14,8),%rbp sbbq 40(%rcx,%r14,8),%rdx - leaq 4(%r14),%r14 - decq %r15 + leaq 4(%r14),%r14 // i++ + decq %r15 // doesn't affect CF jnz L$sub4x - movq %rax,0(%rdi,%r14,8) - movq 32(%rsi,%r14,8),%rax + movq %rax,0(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq 32(%rsi,%r14,8),%rax // load overflow bit sbbq 16(%rcx,%r14,8),%rbx - movq %rdx,8(%rdi,%r14,8) + movq %rdx,8(%rdi,%r14,8) // rp[i]=tp[i]-np[i] sbbq 24(%rcx,%r14,8),%rbp - movq %rbx,16(%rdi,%r14,8) + movq %rbx,16(%rdi,%r14,8) // rp[i]=tp[i]-np[i] - sbbq $0,%rax - movq %rbp,24(%rdi,%r14,8) + sbbq $0,%rax // handle upmost overflow bit + movq %rbp,24(%rdi,%r14,8) // rp[i]=tp[i]-np[i] pxor %xmm0,%xmm0 movq %rax,%xmm4 pcmpeqd %xmm5,%xmm5 pshufd $0,%xmm4,%xmm4 movq %r9,%r15 pxor %xmm4,%xmm5 - shrq $2,%r15 - xorl %eax,%eax + shrq $2,%r15 // j=num/4 + xorl %eax,%eax // i=0 jmp L$copy4x .p2align 4 -L$copy4x: +L$copy4x: // conditional copy movdqa (%rsp,%rax,1),%xmm1 movdqu (%rdi,%rax,1),%xmm2 pand %xmm4,%xmm1 @@ -661,9 +661,9 @@ leaq 32(%rax),%rax decq %r15 jnz L$copy4x - movq 8(%rsp,%r9,8),%rsi + movq 8(%rsp,%r9,8),%rsi // restore %rsp - +// No return value movq -48(%rsi),%r15 movq -40(%rsi),%r14 @@ -682,8 +682,8 @@ ret - - +// see x86_64-mont5 module +// see x86_64-mont5 module .globl _bn_sqr8x_mont .private_extern _bn_sqr8x_mont @@ -711,30 +711,30 @@ L$sqr8x_prologue: movl %r9d,%r10d - shll $3,%r9d - shlq $3+2,%r10 + shll $3,%r9d // convert %r9 to bytes + shlq $3+2,%r10 // 4*%r9 negq %r9 - - - - - +// ############################################################# +// ensure that stack frame doesn't alias with %rsi modulo +// 4096. this is done to allow memory disambiguation logic +// do its job. +// leaq -64(%rsp,%r9,2),%r11 movq %rsp,%rbp - movq (%r8),%r8 + movq (%r8),%r8 // *n0 subq %rsi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb L$sqr8x_sp_alt - subq %r11,%rbp - leaq -64(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rsi + leaq -64(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9) jmp L$sqr8x_sp_done .p2align 5 L$sqr8x_sp_alt: - leaq 4096-64(,%r9,2),%r10 - leaq -64(%rbp,%r9,2),%rbp + leaq 4096-64(,%r9,2),%r10 // 4096-frame-2*%r9 + leaq -64(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -762,41 +762,41 @@ negq %r9 movq %r8,32(%rsp) - movq %rax,40(%rsp) + movq %rax,40(%rsp) // save original %rsp L$sqr8x_body: - movq %rcx,%xmm2 + movq %rcx,%xmm2 // save pointer to modulus pxor %xmm0,%xmm0 - movq %rdi,%xmm1 - movq %r10,%xmm3 + movq %rdi,%xmm1 // save %rdi + movq %r10,%xmm3 // -%r9 testq %rdx,%rdx jz L$sqr8x_nox - call _bn_sqrx8x_internal - - - - + call _bn_sqrx8x_internal // see x86_64-mont5 module +// %rax top-most carry +// %rbp nptr +// %rcx -8*num +// %r8 end of tp[2*num] leaq (%r8,%rcx,1),%rbx movq %rcx,%r9 movq %rcx,%rdx movq %xmm1,%rdi - sarq $3+2,%rcx + sarq $3+2,%rcx // %cf=0 jmp L$sqr8x_sub .p2align 5 L$sqr8x_nox: - call _bn_sqr8x_internal - - - - + call _bn_sqr8x_internal // see x86_64-mont5 module +// %rax top-most carry +// %rbp nptr +// %r8 -8*num +// %rdi end of tp[2*num] leaq (%rdi,%r9,1),%rbx movq %r9,%rcx movq %r9,%rdx movq %xmm1,%rdi - sarq $3+2,%rcx + sarq $3+2,%rcx // %cf=0 jmp L$sqr8x_sub .p2align 5 @@ -816,17 +816,17 @@ movq %r14,16(%rdi) movq %r15,24(%rdi) leaq 32(%rdi),%rdi - incq %rcx + incq %rcx // preserves %cf jnz L$sqr8x_sub - sbbq $0,%rax - leaq (%rbx,%r9,1),%rbx - leaq (%rdi,%r9,1),%rdi + sbbq $0,%rax // top-most carry + leaq (%rbx,%r9,1),%rbx // rewind + leaq (%rdi,%r9,1),%rdi // rewind movq %rax,%xmm1 pxor %xmm0,%xmm0 pshufd $0,%xmm1,%xmm1 - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp jmp L$sqr8x_cond_copy @@ -838,7 +838,7 @@ movdqu 0(%rdi),%xmm4 movdqu 16(%rdi),%xmm5 leaq 32(%rdi),%rdi - movdqa %xmm0,-32(%rbx) + movdqa %xmm0,-32(%rbx) // zero tp movdqa %xmm0,-16(%rbx) movdqa %xmm0,-32(%rbx,%rdx,1) movdqa %xmm0,-16(%rbx,%rdx,1) @@ -855,7 +855,7 @@ addq $32,%r9 jnz L$sqr8x_cond_copy - +// No return value movq -48(%rsi),%r15 movq -40(%rsi),%r14 @@ -898,11 +898,11 @@ L$mulx4x_prologue: - shll $3,%r9d + shll $3,%r9d // convert %r9 to bytes xorq %r10,%r10 - subq %r9,%r10 - movq (%r8),%r8 - leaq -72(%rsp,%r10,1),%rbp + subq %r9,%r10 // -%r9 + movq (%r8),%r8 // *n0 + leaq -72(%rsp,%r10,1),%rbp // future alloca(frame+%r9+8) andq $-128,%rbp movq %rsp,%r11 subq %rbp,%r11 @@ -922,62 +922,62 @@ L$mulx4x_page_walk_done: leaq (%rdx,%r9,1),%r10 - - - - - - - - - - - - - movq %r9,0(%rsp) +// ############################################################# +// Stack layout +// +0 num +// +8 off-loaded &b[i] +// +16 end of b[num] +// +24 saved n0 +// +32 saved rp +// +40 saved %rsp +// +48 inner counter +// +56 +// +64 tmp[num+1] +// + movq %r9,0(%rsp) // save %r9 shrq $5,%r9 - movq %r10,16(%rsp) + movq %r10,16(%rsp) // end of b[num] subq $1,%r9 - movq %r8,24(%rsp) - movq %rdi,32(%rsp) - movq %rax,40(%rsp) + movq %r8,24(%rsp) // save *n0 + movq %rdi,32(%rsp) // save %rdi + movq %rax,40(%rsp) // save original %rsp - movq %r9,48(%rsp) + movq %r9,48(%rsp) // inner counter jmp L$mulx4x_body .p2align 5 L$mulx4x_body: leaq 8(%rdx),%rdi - movq (%rdx),%rdx + movq (%rdx),%rdx // b[0], %rdx==%rdx actually leaq 64+32(%rsp),%rbx movq %rdx,%r9 - mulxq 0(%rsi),%r8,%rax - mulxq 8(%rsi),%r11,%r14 + mulxq 0(%rsi),%r8,%rax // a[0]*b[0] + mulxq 8(%rsi),%r11,%r14 // a[1]*b[0] addq %rax,%r11 - movq %rdi,8(%rsp) - mulxq 16(%rsi),%r12,%r13 + movq %rdi,8(%rsp) // off-load &b[i] + mulxq 16(%rsi),%r12,%r13 // ... adcq %r14,%r12 adcq $0,%r13 - movq %r8,%rdi - imulq 24(%rsp),%r8 - xorq %rbp,%rbp + movq %r8,%rdi // borrow %rdi + imulq 24(%rsp),%r8 // "t[0]"*n0 + xorq %rbp,%rbp // cf=0, of=0 mulxq 24(%rsi),%rax,%r14 movq %r8,%rdx leaq 32(%rsi),%rsi adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%rdi + adcxq %rax,%rdi // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 - movq 48(%rsp),%rdi +.byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 // mulx 2*8(%rcx),%rax,%r12 + movq 48(%rsp),%rdi // counter value movq %r10,-32(%rbx) adcxq %rax,%r11 adoxq %r13,%r12 @@ -985,7 +985,7 @@ movq %r9,%rdx movq %r11,-24(%rbx) adcxq %rax,%r12 - adoxq %rbp,%r15 + adoxq %rbp,%r15 // of=0 leaq 32(%rcx),%rcx movq %r12,-16(%rbx) @@ -993,18 +993,18 @@ .p2align 5 L$mulx4x_1st: - adcxq %rbp,%r15 - mulxq 0(%rsi),%r10,%rax + adcxq %rbp,%r15 // cf=0, modulo-scheduled + mulxq 0(%rsi),%r10,%rax // a[4]*b[0] adcxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[0] adcxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 .byte 0x67,0x67 movq %r8,%rdx adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx @@ -1028,43 +1028,43 @@ leaq 32(%rcx),%rcx movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz L$mulx4x_1st - movq 0(%rsp),%rax - movq 8(%rsp),%rdi - adcq %rbp,%r15 + movq 0(%rsp),%rax // load num + movq 8(%rsp),%rdi // re-load &b[i] + adcq %rbp,%r15 // modulo-scheduled addq %r15,%r14 - sbbq %r15,%r15 + sbbq %r15,%r15 // top-most carry movq %r14,-8(%rbx) jmp L$mulx4x_outer .p2align 5 L$mulx4x_outer: - movq (%rdi),%rdx - leaq 8(%rdi),%rdi - subq %rax,%rsi - movq %r15,(%rbx) + movq (%rdi),%rdx // b[i] + leaq 8(%rdi),%rdi // b++ + subq %rax,%rsi // rewind %rsi + movq %r15,(%rbx) // save top-most carry leaq 64+32(%rsp),%rbx - subq %rax,%rcx + subq %rax,%rcx // rewind %rcx - mulxq 0(%rsi),%r8,%r11 - xorl %ebp,%ebp + mulxq 0(%rsi),%r8,%r11 // a[0]*b[i] + xorl %ebp,%ebp // xor %rbp,%rbp # cf=0, of=0 movq %rdx,%r9 - mulxq 8(%rsi),%r14,%r12 + mulxq 8(%rsi),%r14,%r12 // a[1]*b[i] adoxq -32(%rbx),%r8 adcxq %r14,%r11 - mulxq 16(%rsi),%r15,%r13 + mulxq 16(%rsi),%r15,%r13 // ... adoxq -24(%rbx),%r11 adcxq %r15,%r12 adoxq -16(%rbx),%r12 adcxq %rbp,%r13 adoxq %rbp,%r13 - movq %rdi,8(%rsp) + movq %rdi,8(%rsp) // off-load &b[i] movq %r8,%r15 - imulq 24(%rsp),%r8 - xorl %ebp,%ebp + imulq 24(%rsp),%r8 // "t[0]"*n0 + xorl %ebp,%ebp // xor %rbp,%rbp # cf=0, of=0 mulxq 24(%rsi),%rax,%r14 movq %r8,%rdx @@ -1075,7 +1075,7 @@ adoxq %rbp,%r14 mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%r15 + adcxq %rax,%r15 // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 @@ -1089,21 +1089,21 @@ movq %r11,-24(%rbx) leaq 32(%rcx),%rcx adcxq %rax,%r12 - adoxq %rbp,%r15 - movq 48(%rsp),%rdi + adoxq %rbp,%r15 // of=0 + movq 48(%rsp),%rdi // counter value movq %r12,-16(%rbx) jmp L$mulx4x_inner .p2align 5 L$mulx4x_inner: - mulxq 0(%rsi),%r10,%rax - adcxq %rbp,%r15 + mulxq 0(%rsi),%r10,%rax // a[4]*b[i] + adcxq %rbp,%r15 // cf=0, modulo-scheduled adoxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[i] adcxq 0(%rbx),%r10 adoxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq 8(%rbx),%r11 adoxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 @@ -1111,10 +1111,10 @@ adcxq 16(%rbx),%r12 adoxq %rax,%r13 adcxq 24(%rbx),%r13 - adoxq %rbp,%r14 + adoxq %rbp,%r14 // of=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 adoxq %r15,%r10 mulxq 0(%rcx),%rax,%r15 @@ -1136,26 +1136,26 @@ leaq 32(%rcx),%rcx movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz L$mulx4x_inner - movq 0(%rsp),%rax - movq 8(%rsp),%rdi - adcq %rbp,%r15 - subq 0(%rbx),%rbp + movq 0(%rsp),%rax // load num + movq 8(%rsp),%rdi // re-load &b[i] + adcq %rbp,%r15 // modulo-scheduled + subq 0(%rbx),%rbp // pull top-most carry adcq %r15,%r14 - sbbq %r15,%r15 + sbbq %r15,%r15 // top-most carry movq %r14,-8(%rbx) cmpq 16(%rsp),%rdi jne L$mulx4x_outer leaq 64(%rsp),%rbx - subq %rax,%rcx + subq %rax,%rcx // rewind %rcx negq %r15 movq %rax,%rdx - shrq $3+2,%rax - movq 32(%rsp),%rdi + shrq $3+2,%rax // %cf=0 + movq 32(%rsp),%rdi // restore rp jmp L$mulx4x_sub .p2align 5 @@ -1175,17 +1175,17 @@ movq %r13,16(%rdi) movq %r14,24(%rdi) leaq 32(%rdi),%rdi - decq %rax + decq %rax // preserves %cf jnz L$mulx4x_sub - sbbq $0,%r15 + sbbq $0,%r15 // top-most carry leaq 64(%rsp),%rbx - subq %rdx,%rdi + subq %rdx,%rdi // rewind movq %r15,%xmm1 pxor %xmm0,%xmm0 pshufd $0,%xmm1,%xmm1 - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp jmp L$mulx4x_cond_copy @@ -1197,7 +1197,7 @@ movdqu 0(%rdi),%xmm4 movdqu 16(%rdi),%xmm5 leaq 32(%rdi),%rdi - movdqa %xmm0,-32(%rbx) + movdqa %xmm0,-32(%rbx) // zero tp movdqa %xmm0,-16(%rbx) pcmpeqd %xmm1,%xmm0 pand %xmm1,%xmm2 @@ -1214,7 +1214,7 @@ movq %rdx,(%rbx) - +// No return value movq -48(%rsi),%r15 movq -40(%rsi),%r14
diff --git a/gen/bcm/x86_64-mont-linux.S b/gen/bcm/x86_64-mont-linux.S index 51c4b6c..b8630ea 100644 --- a/gen/bcm/x86_64-mont-linux.S +++ b/gen/bcm/x86_64-mont-linux.S
@@ -31,18 +31,18 @@ negq %r9 movq %rsp,%r11 - leaq -16(%rsp,%r9,8),%r10 - negq %r9 - andq $-1024,%r10 + leaq -16(%rsp,%r9,8),%r10 // future alloca(8*(num+2)) + negq %r9 // restore %r9 + andq $-1024,%r10 // minimize TLB usage - - - - - - - - +// An OS-agnostic version of __chkstk. +// +// Some OSes (Windows) insist on stack being "wired" to +// physical memory in strictly sequential manner, i.e. if stack +// allocation spans two pages, then reference to farmost one can +// be punishable by SEGV. But page walking can do good even on +// other OSes, because it guarantees that villain thread hits +// the guard page before it can make damage to innocent one... subq %r10,%r11 andq $-4096,%r11 leaq (%r10,%r11,1),%rsp @@ -59,32 +59,32 @@ ja .Lmul_page_walk .Lmul_page_walk_done: - movq %rax,8(%rsp,%r9,8) + movq %rax,8(%rsp,%r9,8) // tp[num+1]=%rsp .cfi_escape 0x0f,0x0a,0x77,0x08,0x79,0x00,0x38,0x1e,0x22,0x06,0x23,0x08 .Lmul_body: - movq %rdx,%r12 - movq (%r8),%r8 - movq (%r12),%rbx + movq %rdx,%r12 // reassign %rdx + movq (%r8),%r8 // pull n0[0] value + movq (%r12),%rbx // m0=bp[0] movq (%rsi),%rax - xorq %r14,%r14 - xorq %r15,%r15 + xorq %r14,%r14 // i=0 + xorq %r15,%r15 // j=0 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp .L1st_enter .align 16 @@ -92,30 +92,30 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] movq %r10,%r11 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 .L1st_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 jne .L1st addq %rax,%r13 - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 movq %r10,%r11 @@ -123,32 +123,32 @@ addq %r11,%r13 adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ jmp .Louter .align 16 .Louter: - movq (%r12,%r14,8),%rbx - xorq %r15,%r15 + movq (%r12,%r14,8),%rbx // m0=bp[i] + xorq %r15,%r15 // j=0 movq %r8,%rbp movq (%rsp),%r10 - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx - movq 8(%rsp),%r10 + movq 8(%rsp),%r10 // tp[1] movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp .Linner_enter .align 16 @@ -156,80 +156,80 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r15,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 .Linner_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - addq %r11,%r10 + addq %r11,%r10 // ap[j]*bp[i]+tp[j] movq %rdx,%r11 adcq $0,%r11 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 jne .Linner addq %rax,%r13 - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r15,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 xorq %rdx,%rdx addq %r11,%r13 adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // pull upmost overflow bit adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ cmpq %r9,%r14 jb .Louter - xorq %r14,%r14 - movq (%rsp),%rax - movq %r9,%r15 + xorq %r14,%r14 // i=0 and clear CF + movq (%rsp),%rax // tp[0] + movq %r9,%r15 // j=num .align 16 .Lsub: sbbq (%rcx,%r14,8),%rax - movq %rax,(%rdi,%r14,8) - movq 8(%rsp,%r14,8),%rax - leaq 1(%r14),%r14 - decq %r15 + movq %rax,(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq 8(%rsp,%r14,8),%rax // tp[i+1] + leaq 1(%r14),%r14 // i++ + decq %r15 // doesn't affect CF jnz .Lsub - sbbq $0,%rax + sbbq $0,%rax // handle upmost overflow bit movq $-1,%rbx - xorq %rax,%rbx + xorq %rax,%rbx // not %rax xorq %r14,%r14 - movq %r9,%r15 + movq %r9,%r15 // j=num -.Lcopy: +.Lcopy: // conditional copy movq (%rdi,%r14,8),%rcx movq (%rsp,%r14,8),%rdx andq %rbx,%rcx andq %rax,%rdx - movq %r9,(%rsp,%r14,8) + movq %r9,(%rsp,%r14,8) // zap temporary vector orq %rcx,%rdx - movq %rdx,(%rdi,%r14,8) + movq %rdx,(%rdi,%r14,8) // rp[i]=tp[i] leaq 1(%r14),%r14 subq $1,%r15 jnz .Lcopy - movq 8(%rsp,%r9,8),%rsi + movq 8(%rsp,%r9,8),%rsi // restore %rsp .cfi_def_cfa %rsi,8 - +// No return value movq -48(%rsi),%r15 .cfi_restore %r15 movq -40(%rsi),%r14 @@ -273,9 +273,9 @@ negq %r9 movq %rsp,%r11 - leaq -32(%rsp,%r9,8),%r10 - negq %r9 - andq $-1024,%r10 + leaq -32(%rsp,%r9,8),%r10 // future alloca(8*(num+4)) + negq %r9 // restore + andq $-1024,%r10 // minimize TLB usage subq %r10,%r11 andq $-4096,%r11 @@ -292,28 +292,28 @@ ja .Lmul4x_page_walk .Lmul4x_page_walk_done: - movq %rax,8(%rsp,%r9,8) + movq %rax,8(%rsp,%r9,8) // tp[num+1]=%rsp .cfi_escape 0x0f,0x0a,0x77,0x08,0x79,0x00,0x38,0x1e,0x22,0x06,0x23,0x08 .Lmul4x_body: - movq %rdi,16(%rsp,%r9,8) - movq %rdx,%r12 - movq (%r8),%r8 - movq (%r12),%rbx + movq %rdi,16(%rsp,%r9,8) // tp[num+2]=%rdi + movq %rdx,%r12 // reassign %r12 + movq (%r8),%r8 // pull n0[0] value + movq (%r12),%rbx // m0=bp[0] movq (%rsi),%rax - xorq %r14,%r14 - xorq %r15,%r15 + xorq %r14,%r14 // i=0 + xorq %r15,%r15 // j=0 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%rdi @@ -329,171 +329,171 @@ movq 16(%rsi),%rax adcq $0,%rdx addq %r11,%rdi - leaq 4(%r15),%r15 + leaq 4(%r15),%r15 // j++ adcq $0,%rdx movq %rdi,(%rsp) movq %rdx,%r13 jmp .L1st4x .align 16 .L1st4x: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq (%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-8(%rsp,%r15,8) + movq %r13,-8(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq 8(%rcx,%r15,8),%rax adcq $0,%rdx - leaq 4(%r15),%r15 + leaq 4(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq -16(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-32(%rsp,%r15,8) + movq %rdi,-32(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 cmpq %r9,%r15 jb .L1st4x - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 xorq %rdi,%rdi addq %r10,%r13 adcq $0,%rdi movq %r13,-8(%rsp,%r15,8) - movq %rdi,(%rsp,%r15,8) + movq %rdi,(%rsp,%r15,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ .align 4 .Louter4x: - movq (%r12,%r14,8),%rbx - xorq %r15,%r15 + movq (%r12,%r14,8),%rbx // m0=bp[i] + xorq %r15,%r15 // j=0 movq (%rsp),%r10 movq %r8,%rbp - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // "%r13", discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx - addq 8(%rsp),%r11 + addq 8(%rsp),%r11 // +tp[1] adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi),%rax adcq $0,%rdx - addq %r11,%rdi - leaq 4(%r15),%r15 + addq %r11,%rdi // np[j]*m1+ap[j]*bp[i]+tp[j] + leaq 4(%r15),%r15 // j+=2 adcq $0,%rdx - movq %rdi,(%rsp) + movq %rdi,(%rsp) // tp[j-1] movq %rdx,%r13 jmp .Linner4x .align 16 .Linner4x: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx - addq -16(%rsp,%r15,8),%r10 + addq -16(%rsp,%r15,8),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx @@ -501,148 +501,148 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,8),%rax adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq (%rcx,%r15,8),%rax adcq $0,%rdx - addq (%rsp,%r15,8),%r10 + addq (%rsp,%r15,8),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,8),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %r13,-8(%rsp,%r15,8) + movq %r13,-8(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx,%r15,8),%rax adcq $0,%rdx addq 8(%rsp,%r15,8),%r11 adcq $0,%rdx - leaq 4(%r15),%r15 + leaq 4(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq -16(%rsi,%r15,8),%rax adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %rdi,-32(%rsp,%r15,8) + movq %rdi,-32(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 cmpq %r9,%r15 jb .Linner4x - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx,%r15,8),%rax adcq $0,%rdx - addq -16(%rsp,%r15,8),%r10 + addq -16(%rsp,%r15,8),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,8),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %r13,-24(%rsp,%r15,8) + movq %r13,-24(%rsp,%r15,8) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq -8(%rcx,%r15,8),%rax adcq $0,%rdx addq -8(%rsp,%r15,8),%r11 adcq $0,%rdx - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi),%rax + movq (%rsi),%rax // ap[0] adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %rdi,-16(%rsp,%r15,8) + movq %rdi,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 xorq %rdi,%rdi addq %r10,%r13 adcq $0,%rdi - addq (%rsp,%r9,8),%r13 + addq (%rsp,%r9,8),%r13 // pull upmost overflow bit adcq $0,%rdi movq %r13,-8(%rsp,%r15,8) - movq %rdi,(%rsp,%r15,8) + movq %rdi,(%rsp,%r15,8) // store upmost overflow bit cmpq %r9,%r14 jb .Louter4x - movq 16(%rsp,%r9,8),%rdi + movq 16(%rsp,%r9,8),%rdi // restore %rdi leaq -4(%r9),%r15 - movq 0(%rsp),%rax - movq 8(%rsp),%rdx - shrq $2,%r15 - leaq (%rsp),%rsi - xorq %r14,%r14 + movq 0(%rsp),%rax // tp[0] + movq 8(%rsp),%rdx // tp[1] + shrq $2,%r15 // j=num/4-1 + leaq (%rsp),%rsi // borrow ap for tp + xorq %r14,%r14 // i=0 and clear CF subq 0(%rcx),%rax - movq 16(%rsi),%rbx - movq 24(%rsi),%rbp + movq 16(%rsi),%rbx // tp[2] + movq 24(%rsi),%rbp // tp[3] sbbq 8(%rcx),%rdx .Lsub4x: - movq %rax,0(%rdi,%r14,8) - movq %rdx,8(%rdi,%r14,8) + movq %rax,0(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq %rdx,8(%rdi,%r14,8) // rp[i]=tp[i]-np[i] sbbq 16(%rcx,%r14,8),%rbx - movq 32(%rsi,%r14,8),%rax + movq 32(%rsi,%r14,8),%rax // tp[i+1] movq 40(%rsi,%r14,8),%rdx sbbq 24(%rcx,%r14,8),%rbp - movq %rbx,16(%rdi,%r14,8) - movq %rbp,24(%rdi,%r14,8) + movq %rbx,16(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq %rbp,24(%rdi,%r14,8) // rp[i]=tp[i]-np[i] sbbq 32(%rcx,%r14,8),%rax movq 48(%rsi,%r14,8),%rbx movq 56(%rsi,%r14,8),%rbp sbbq 40(%rcx,%r14,8),%rdx - leaq 4(%r14),%r14 - decq %r15 + leaq 4(%r14),%r14 // i++ + decq %r15 // doesn't affect CF jnz .Lsub4x - movq %rax,0(%rdi,%r14,8) - movq 32(%rsi,%r14,8),%rax + movq %rax,0(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq 32(%rsi,%r14,8),%rax // load overflow bit sbbq 16(%rcx,%r14,8),%rbx - movq %rdx,8(%rdi,%r14,8) + movq %rdx,8(%rdi,%r14,8) // rp[i]=tp[i]-np[i] sbbq 24(%rcx,%r14,8),%rbp - movq %rbx,16(%rdi,%r14,8) + movq %rbx,16(%rdi,%r14,8) // rp[i]=tp[i]-np[i] - sbbq $0,%rax - movq %rbp,24(%rdi,%r14,8) + sbbq $0,%rax // handle upmost overflow bit + movq %rbp,24(%rdi,%r14,8) // rp[i]=tp[i]-np[i] pxor %xmm0,%xmm0 movq %rax,%xmm4 pcmpeqd %xmm5,%xmm5 pshufd $0,%xmm4,%xmm4 movq %r9,%r15 pxor %xmm4,%xmm5 - shrq $2,%r15 - xorl %eax,%eax + shrq $2,%r15 // j=num/4 + xorl %eax,%eax // i=0 jmp .Lcopy4x .align 16 -.Lcopy4x: +.Lcopy4x: // conditional copy movdqa (%rsp,%rax,1),%xmm1 movdqu (%rdi,%rax,1),%xmm2 pand %xmm4,%xmm1 @@ -660,9 +660,9 @@ leaq 32(%rax),%rax decq %r15 jnz .Lcopy4x - movq 8(%rsp,%r9,8),%rsi + movq 8(%rsp,%r9,8),%rsi // restore %rsp .cfi_def_cfa %rsi, 8 - +// No return value movq -48(%rsi),%r15 .cfi_restore %r15 movq -40(%rsi),%r14 @@ -682,9 +682,9 @@ .cfi_endproc .size bn_mul4x_mont,.-bn_mul4x_mont .extern bn_sqrx8x_internal -.hidden bn_sqrx8x_internal +.hidden bn_sqrx8x_internal // see x86_64-mont5 module .extern bn_sqr8x_internal -.hidden bn_sqr8x_internal +.hidden bn_sqr8x_internal // see x86_64-mont5 module .globl bn_sqr8x_mont .hidden bn_sqr8x_mont @@ -711,30 +711,30 @@ .Lsqr8x_prologue: movl %r9d,%r10d - shll $3,%r9d - shlq $3+2,%r10 + shll $3,%r9d // convert %r9 to bytes + shlq $3+2,%r10 // 4*%r9 negq %r9 - - - - - +// ############################################################# +// ensure that stack frame doesn't alias with %rsi modulo +// 4096. this is done to allow memory disambiguation logic +// do its job. +// leaq -64(%rsp,%r9,2),%r11 movq %rsp,%rbp - movq (%r8),%r8 + movq (%r8),%r8 // *n0 subq %rsi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb .Lsqr8x_sp_alt - subq %r11,%rbp - leaq -64(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rsi + leaq -64(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9) jmp .Lsqr8x_sp_done .align 32 .Lsqr8x_sp_alt: - leaq 4096-64(,%r9,2),%r10 - leaq -64(%rbp,%r9,2),%rbp + leaq 4096-64(,%r9,2),%r10 // 4096-frame-2*%r9 + leaq -64(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -762,41 +762,41 @@ negq %r9 movq %r8,32(%rsp) - movq %rax,40(%rsp) + movq %rax,40(%rsp) // save original %rsp .cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08 .Lsqr8x_body: - movq %rcx,%xmm2 + movq %rcx,%xmm2 // save pointer to modulus pxor %xmm0,%xmm0 - movq %rdi,%xmm1 - movq %r10,%xmm3 + movq %rdi,%xmm1 // save %rdi + movq %r10,%xmm3 // -%r9 testq %rdx,%rdx jz .Lsqr8x_nox - call bn_sqrx8x_internal - - - - + call bn_sqrx8x_internal // see x86_64-mont5 module +// %rax top-most carry +// %rbp nptr +// %rcx -8*num +// %r8 end of tp[2*num] leaq (%r8,%rcx,1),%rbx movq %rcx,%r9 movq %rcx,%rdx movq %xmm1,%rdi - sarq $3+2,%rcx + sarq $3+2,%rcx // %cf=0 jmp .Lsqr8x_sub .align 32 .Lsqr8x_nox: - call bn_sqr8x_internal - - - - + call bn_sqr8x_internal // see x86_64-mont5 module +// %rax top-most carry +// %rbp nptr +// %r8 -8*num +// %rdi end of tp[2*num] leaq (%rdi,%r9,1),%rbx movq %r9,%rcx movq %r9,%rdx movq %xmm1,%rdi - sarq $3+2,%rcx + sarq $3+2,%rcx // %cf=0 jmp .Lsqr8x_sub .align 32 @@ -816,17 +816,17 @@ movq %r14,16(%rdi) movq %r15,24(%rdi) leaq 32(%rdi),%rdi - incq %rcx + incq %rcx // preserves %cf jnz .Lsqr8x_sub - sbbq $0,%rax - leaq (%rbx,%r9,1),%rbx - leaq (%rdi,%r9,1),%rdi + sbbq $0,%rax // top-most carry + leaq (%rbx,%r9,1),%rbx // rewind + leaq (%rdi,%r9,1),%rdi // rewind movq %rax,%xmm1 pxor %xmm0,%xmm0 pshufd $0,%xmm1,%xmm1 - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp .cfi_def_cfa %rsi,8 jmp .Lsqr8x_cond_copy @@ -838,7 +838,7 @@ movdqu 0(%rdi),%xmm4 movdqu 16(%rdi),%xmm5 leaq 32(%rdi),%rdi - movdqa %xmm0,-32(%rbx) + movdqa %xmm0,-32(%rbx) // zero tp movdqa %xmm0,-16(%rbx) movdqa %xmm0,-32(%rbx,%rdx,1) movdqa %xmm0,-16(%rbx,%rdx,1) @@ -855,7 +855,7 @@ addq $32,%r9 jnz .Lsqr8x_cond_copy - +// No return value movq -48(%rsi),%r15 .cfi_restore %r15 movq -40(%rsi),%r14 @@ -897,11 +897,11 @@ .cfi_offset %r15,-56 .Lmulx4x_prologue: - shll $3,%r9d + shll $3,%r9d // convert %r9 to bytes xorq %r10,%r10 - subq %r9,%r10 - movq (%r8),%r8 - leaq -72(%rsp,%r10,1),%rbp + subq %r9,%r10 // -%r9 + movq (%r8),%r8 // *n0 + leaq -72(%rsp,%r10,1),%rbp // future alloca(frame+%r9+8) andq $-128,%rbp movq %rsp,%r11 subq %rbp,%r11 @@ -921,62 +921,62 @@ .Lmulx4x_page_walk_done: leaq (%rdx,%r9,1),%r10 - - - - - - - - - - - - - movq %r9,0(%rsp) +// ############################################################# +// Stack layout +// +0 num +// +8 off-loaded &b[i] +// +16 end of b[num] +// +24 saved n0 +// +32 saved rp +// +40 saved %rsp +// +48 inner counter +// +56 +// +64 tmp[num+1] +// + movq %r9,0(%rsp) // save %r9 shrq $5,%r9 - movq %r10,16(%rsp) + movq %r10,16(%rsp) // end of b[num] subq $1,%r9 - movq %r8,24(%rsp) - movq %rdi,32(%rsp) - movq %rax,40(%rsp) + movq %r8,24(%rsp) // save *n0 + movq %rdi,32(%rsp) // save %rdi + movq %rax,40(%rsp) // save original %rsp .cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08 - movq %r9,48(%rsp) + movq %r9,48(%rsp) // inner counter jmp .Lmulx4x_body .align 32 .Lmulx4x_body: leaq 8(%rdx),%rdi - movq (%rdx),%rdx + movq (%rdx),%rdx // b[0], %rdx==%rdx actually leaq 64+32(%rsp),%rbx movq %rdx,%r9 - mulxq 0(%rsi),%r8,%rax - mulxq 8(%rsi),%r11,%r14 + mulxq 0(%rsi),%r8,%rax // a[0]*b[0] + mulxq 8(%rsi),%r11,%r14 // a[1]*b[0] addq %rax,%r11 - movq %rdi,8(%rsp) - mulxq 16(%rsi),%r12,%r13 + movq %rdi,8(%rsp) // off-load &b[i] + mulxq 16(%rsi),%r12,%r13 // ... adcq %r14,%r12 adcq $0,%r13 - movq %r8,%rdi - imulq 24(%rsp),%r8 - xorq %rbp,%rbp + movq %r8,%rdi // borrow %rdi + imulq 24(%rsp),%r8 // "t[0]"*n0 + xorq %rbp,%rbp // cf=0, of=0 mulxq 24(%rsi),%rax,%r14 movq %r8,%rdx leaq 32(%rsi),%rsi adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%rdi + adcxq %rax,%rdi // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 - movq 48(%rsp),%rdi +.byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 // mulx 2*8(%rcx),%rax,%r12 + movq 48(%rsp),%rdi // counter value movq %r10,-32(%rbx) adcxq %rax,%r11 adoxq %r13,%r12 @@ -984,7 +984,7 @@ movq %r9,%rdx movq %r11,-24(%rbx) adcxq %rax,%r12 - adoxq %rbp,%r15 + adoxq %rbp,%r15 // of=0 leaq 32(%rcx),%rcx movq %r12,-16(%rbx) @@ -992,18 +992,18 @@ .align 32 .Lmulx4x_1st: - adcxq %rbp,%r15 - mulxq 0(%rsi),%r10,%rax + adcxq %rbp,%r15 // cf=0, modulo-scheduled + mulxq 0(%rsi),%r10,%rax // a[4]*b[0] adcxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[0] adcxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 .byte 0x67,0x67 movq %r8,%rdx adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx @@ -1027,43 +1027,43 @@ leaq 32(%rcx),%rcx movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz .Lmulx4x_1st - movq 0(%rsp),%rax - movq 8(%rsp),%rdi - adcq %rbp,%r15 + movq 0(%rsp),%rax // load num + movq 8(%rsp),%rdi // re-load &b[i] + adcq %rbp,%r15 // modulo-scheduled addq %r15,%r14 - sbbq %r15,%r15 + sbbq %r15,%r15 // top-most carry movq %r14,-8(%rbx) jmp .Lmulx4x_outer .align 32 .Lmulx4x_outer: - movq (%rdi),%rdx - leaq 8(%rdi),%rdi - subq %rax,%rsi - movq %r15,(%rbx) + movq (%rdi),%rdx // b[i] + leaq 8(%rdi),%rdi // b++ + subq %rax,%rsi // rewind %rsi + movq %r15,(%rbx) // save top-most carry leaq 64+32(%rsp),%rbx - subq %rax,%rcx + subq %rax,%rcx // rewind %rcx - mulxq 0(%rsi),%r8,%r11 - xorl %ebp,%ebp + mulxq 0(%rsi),%r8,%r11 // a[0]*b[i] + xorl %ebp,%ebp // xor %rbp,%rbp # cf=0, of=0 movq %rdx,%r9 - mulxq 8(%rsi),%r14,%r12 + mulxq 8(%rsi),%r14,%r12 // a[1]*b[i] adoxq -32(%rbx),%r8 adcxq %r14,%r11 - mulxq 16(%rsi),%r15,%r13 + mulxq 16(%rsi),%r15,%r13 // ... adoxq -24(%rbx),%r11 adcxq %r15,%r12 adoxq -16(%rbx),%r12 adcxq %rbp,%r13 adoxq %rbp,%r13 - movq %rdi,8(%rsp) + movq %rdi,8(%rsp) // off-load &b[i] movq %r8,%r15 - imulq 24(%rsp),%r8 - xorl %ebp,%ebp + imulq 24(%rsp),%r8 // "t[0]"*n0 + xorl %ebp,%ebp // xor %rbp,%rbp # cf=0, of=0 mulxq 24(%rsi),%rax,%r14 movq %r8,%rdx @@ -1074,7 +1074,7 @@ adoxq %rbp,%r14 mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%r15 + adcxq %rax,%r15 // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 @@ -1088,21 +1088,21 @@ movq %r11,-24(%rbx) leaq 32(%rcx),%rcx adcxq %rax,%r12 - adoxq %rbp,%r15 - movq 48(%rsp),%rdi + adoxq %rbp,%r15 // of=0 + movq 48(%rsp),%rdi // counter value movq %r12,-16(%rbx) jmp .Lmulx4x_inner .align 32 .Lmulx4x_inner: - mulxq 0(%rsi),%r10,%rax - adcxq %rbp,%r15 + mulxq 0(%rsi),%r10,%rax // a[4]*b[i] + adcxq %rbp,%r15 // cf=0, modulo-scheduled adoxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[i] adcxq 0(%rbx),%r10 adoxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq 8(%rbx),%r11 adoxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 @@ -1110,10 +1110,10 @@ adcxq 16(%rbx),%r12 adoxq %rax,%r13 adcxq 24(%rbx),%r13 - adoxq %rbp,%r14 + adoxq %rbp,%r14 // of=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 adoxq %r15,%r10 mulxq 0(%rcx),%rax,%r15 @@ -1135,26 +1135,26 @@ leaq 32(%rcx),%rcx movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz .Lmulx4x_inner - movq 0(%rsp),%rax - movq 8(%rsp),%rdi - adcq %rbp,%r15 - subq 0(%rbx),%rbp + movq 0(%rsp),%rax // load num + movq 8(%rsp),%rdi // re-load &b[i] + adcq %rbp,%r15 // modulo-scheduled + subq 0(%rbx),%rbp // pull top-most carry adcq %r15,%r14 - sbbq %r15,%r15 + sbbq %r15,%r15 // top-most carry movq %r14,-8(%rbx) cmpq 16(%rsp),%rdi jne .Lmulx4x_outer leaq 64(%rsp),%rbx - subq %rax,%rcx + subq %rax,%rcx // rewind %rcx negq %r15 movq %rax,%rdx - shrq $3+2,%rax - movq 32(%rsp),%rdi + shrq $3+2,%rax // %cf=0 + movq 32(%rsp),%rdi // restore rp jmp .Lmulx4x_sub .align 32 @@ -1174,17 +1174,17 @@ movq %r13,16(%rdi) movq %r14,24(%rdi) leaq 32(%rdi),%rdi - decq %rax + decq %rax // preserves %cf jnz .Lmulx4x_sub - sbbq $0,%r15 + sbbq $0,%r15 // top-most carry leaq 64(%rsp),%rbx - subq %rdx,%rdi + subq %rdx,%rdi // rewind movq %r15,%xmm1 pxor %xmm0,%xmm0 pshufd $0,%xmm1,%xmm1 - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp .cfi_def_cfa %rsi,8 jmp .Lmulx4x_cond_copy @@ -1196,7 +1196,7 @@ movdqu 0(%rdi),%xmm4 movdqu 16(%rdi),%xmm5 leaq 32(%rdi),%rdi - movdqa %xmm0,-32(%rbx) + movdqa %xmm0,-32(%rbx) // zero tp movdqa %xmm0,-16(%rbx) pcmpeqd %xmm1,%xmm0 pand %xmm1,%xmm2 @@ -1213,7 +1213,7 @@ movq %rdx,(%rbx) - +// No return value movq -48(%rsi),%r15 .cfi_restore %r15 movq -40(%rsi),%r14
diff --git a/gen/bcm/x86_64-mont-win.asm b/gen/bcm/x86_64-mont-win.asm index 721d5dc..5ff35fe 100644 --- a/gen/bcm/x86_64-mont-win.asm +++ b/gen/bcm/x86_64-mont-win.asm
@@ -50,18 +50,18 @@ neg r9 mov r11,rsp - lea r10,[((-16))+r9*8+rsp] - neg r9 - and r10,-1024 + lea r10,[((-16))+r9*8+rsp] ; future alloca(8*(num+2)) + neg r9 ; restore %r9 + and r10,-1024 ; minimize TLB usage - - - - - - - - +; An OS-agnostic version of __chkstk. +; +; Some OSes (Windows) insist on stack being "wired" to +; physical memory in strictly sequential manner, i.e. if stack +; allocation spans two pages, then reference to farmost one can +; be punishable by SEGV. But page walking can do good even on +; other OSes, because it guarantees that villain thread hits +; the guard page before it can make damage to innocent one... sub r11,r10 and r11,-4096 lea rsp,[r11*1+r10] @@ -78,32 +78,32 @@ ja NEAR $L$mul_page_walk $L$mul_page_walk_done: - mov QWORD[8+r9*8+rsp],rax + mov QWORD[8+r9*8+rsp],rax ; tp[num+1]=%rsp $L$mul_body: - mov r12,rdx - mov r8,QWORD[r8] - mov rbx,QWORD[r12] + mov r12,rdx ; reassign %rdx + mov r8,QWORD[r8] ; pull n0[0] value + mov rbx,QWORD[r12] ; m0=bp[0] mov rax,QWORD[rsi] - xor r14,r14 - xor r15,r15 + xor r14,r14 ; i=0 + xor r15,r15 ; j=0 mov rbp,r8 - mul rbx + mul rbx ; ap[0]*bp[0] mov r10,rax mov rax,QWORD[rcx] - imul rbp,r10 + imul rbp,r10 ; "tp[0]"*n0 mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; discarded mov rax,QWORD[8+rsi] adc rdx,0 mov r13,rdx - lea r15,[1+r15] + lea r15,[1+r15] ; j++ jmp NEAR $L$1st_enter ALIGN 16 @@ -111,30 +111,30 @@ add r13,rax mov rax,QWORD[r15*8+rsi] adc rdx,0 - add r13,r11 + add r13,r11 ; np[j]*m1+ap[j]*bp[0] mov r11,r10 adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],r13 + mov QWORD[((-16))+r15*8+rsp],r13 ; tp[j-1] mov r13,rdx $L$1st_enter: - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[r15*8+rcx] adc rdx,0 - lea r15,[1+r15] + lea r15,[1+r15] ; j++ mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 cmp r15,r9 jne NEAR $L$1st add r13,rax - mov rax,QWORD[rsi] + mov rax,QWORD[rsi] ; ap[0] adc rdx,0 - add r13,r11 + add r13,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],r13 + mov QWORD[((-16))+r15*8+rsp],r13 ; tp[j-1] mov r13,rdx mov r11,r10 @@ -142,32 +142,32 @@ add r13,r11 adc rdx,0 mov QWORD[((-8))+r9*8+rsp],r13 - mov QWORD[r9*8+rsp],rdx + mov QWORD[r9*8+rsp],rdx ; store upmost overflow bit - lea r14,[1+r14] + lea r14,[1+r14] ; i++ jmp NEAR $L$outer ALIGN 16 $L$outer: - mov rbx,QWORD[r14*8+r12] - xor r15,r15 + mov rbx,QWORD[r14*8+r12] ; m0=bp[i] + xor r15,r15 ; j=0 mov rbp,r8 mov r10,QWORD[rsp] - mul rbx - add r10,rax + mul rbx ; ap[0]*bp[i] + add r10,rax ; ap[0]*bp[i]+tp[0] mov rax,QWORD[rcx] adc rdx,0 - imul rbp,r10 + imul rbp,r10 ; tp[0]*n0 mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; discarded mov rax,QWORD[8+rsi] adc rdx,0 - mov r10,QWORD[8+rsp] + mov r10,QWORD[8+rsp] ; tp[1] mov r13,rdx - lea r15,[1+r15] + lea r15,[1+r15] ; j++ jmp NEAR $L$inner_enter ALIGN 16 @@ -175,80 +175,80 @@ add r13,rax mov rax,QWORD[r15*8+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[i]+tp[j] mov r10,QWORD[r15*8+rsp] adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],r13 + mov QWORD[((-16))+r15*8+rsp],r13 ; tp[j-1] mov r13,rdx $L$inner_enter: - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[r15*8+rcx] adc rdx,0 - add r10,r11 + add r10,r11 ; ap[j]*bp[i]+tp[j] mov r11,rdx adc r11,0 - lea r15,[1+r15] + lea r15,[1+r15] ; j++ - mul rbp + mul rbp ; np[j]*m1 cmp r15,r9 jne NEAR $L$inner add r13,rax - mov rax,QWORD[rsi] + mov rax,QWORD[rsi] ; ap[0] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[i]+tp[j] mov r10,QWORD[r15*8+rsp] adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],r13 + mov QWORD[((-16))+r15*8+rsp],r13 ; tp[j-1] mov r13,rdx xor rdx,rdx add r13,r11 adc rdx,0 - add r13,r10 + add r13,r10 ; pull upmost overflow bit adc rdx,0 mov QWORD[((-8))+r9*8+rsp],r13 - mov QWORD[r9*8+rsp],rdx + mov QWORD[r9*8+rsp],rdx ; store upmost overflow bit - lea r14,[1+r14] + lea r14,[1+r14] ; i++ cmp r14,r9 jb NEAR $L$outer - xor r14,r14 - mov rax,QWORD[rsp] - mov r15,r9 + xor r14,r14 ; i=0 and clear CF! + mov rax,QWORD[rsp] ; tp[0] + mov r15,r9 ; j=num ALIGN 16 $L$sub: sbb rax,QWORD[r14*8+rcx] - mov QWORD[r14*8+rdi],rax - mov rax,QWORD[8+r14*8+rsp] - lea r14,[1+r14] - dec r15 + mov QWORD[r14*8+rdi],rax ; rp[i]=tp[i]-np[i] + mov rax,QWORD[8+r14*8+rsp] ; tp[i+1] + lea r14,[1+r14] ; i++ + dec r15 ; doesn't affect CF! jnz NEAR $L$sub - sbb rax,0 + sbb rax,0 ; handle upmost overflow bit mov rbx,-1 - xor rbx,rax + xor rbx,rax ; not %rax xor r14,r14 - mov r15,r9 + mov r15,r9 ; j=num -$L$copy: +$L$copy: ; conditional copy mov rcx,QWORD[r14*8+rdi] mov rdx,QWORD[r14*8+rsp] and rcx,rbx and rdx,rax - mov QWORD[r14*8+rsp],r9 + mov QWORD[r14*8+rsp],r9 ; zap temporary vector or rdx,rcx - mov QWORD[r14*8+rdi],rdx + mov QWORD[r14*8+rdi],rdx ; rp[i]=tp[i] lea r14,[1+r14] sub r15,1 jnz NEAR $L$copy - mov rsi,QWORD[8+r9*8+rsp] + mov rsi,QWORD[8+r9*8+rsp] ; restore %rsp - +; No return value mov r15,QWORD[((-48))+rsi] mov r14,QWORD[((-40))+rsi] @@ -305,9 +305,9 @@ neg r9 mov r11,rsp - lea r10,[((-32))+r9*8+rsp] - neg r9 - and r10,-1024 + lea r10,[((-32))+r9*8+rsp] ; future alloca(8*(num+4)) + neg r9 ; restore + and r10,-1024 ; minimize TLB usage sub r11,r10 and r11,-4096 @@ -324,28 +324,28 @@ ja NEAR $L$mul4x_page_walk $L$mul4x_page_walk_done: - mov QWORD[8+r9*8+rsp],rax + mov QWORD[8+r9*8+rsp],rax ; tp[num+1]=%rsp $L$mul4x_body: - mov QWORD[16+r9*8+rsp],rdi - mov r12,rdx - mov r8,QWORD[r8] - mov rbx,QWORD[r12] + mov QWORD[16+r9*8+rsp],rdi ; tp[num+2]=%rdi + mov r12,rdx ; reassign %r12 + mov r8,QWORD[r8] ; pull n0[0] value + mov rbx,QWORD[r12] ; m0=bp[0] mov rax,QWORD[rsi] - xor r14,r14 - xor r15,r15 + xor r14,r14 ; i=0 + xor r15,r15 ; j=0 mov rbp,r8 - mul rbx + mul rbx ; ap[0]*bp[0] mov r10,rax mov rax,QWORD[rcx] - imul rbp,r10 + imul rbp,r10 ; "tp[0]"*n0 mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; discarded mov rax,QWORD[8+rsi] adc rdx,0 mov rdi,rdx @@ -361,171 +361,171 @@ mov rax,QWORD[16+rsi] adc rdx,0 add rdi,r11 - lea r15,[4+r15] + lea r15,[4+r15] ; j++ adc rdx,0 mov QWORD[rsp],rdi mov r13,rdx jmp NEAR $L$1st4x ALIGN 16 $L$1st4x: - mul rbx + mul rbx ; ap[j]*bp[0] add r10,rax mov rax,QWORD[((-16))+r15*8+rcx] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+r15*8+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-24))+r15*8+rsp],r13 + mov QWORD[((-24))+r15*8+rsp],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[((-8))+r15*8+rcx] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[r15*8+rsi] adc rdx,0 - add rdi,r11 + add rdi,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],rdi + mov QWORD[((-16))+r15*8+rsp],rdi ; tp[j-1] mov r13,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r10,rax mov rax,QWORD[r15*8+rcx] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[8+r15*8+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-8))+r15*8+rsp],r13 + mov QWORD[((-8))+r15*8+rsp],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[8+r15*8+rcx] adc rdx,0 - lea r15,[4+r15] + lea r15,[4+r15] ; j++ mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[((-16))+r15*8+rsi] adc rdx,0 - add rdi,r11 + add rdi,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-32))+r15*8+rsp],rdi + mov QWORD[((-32))+r15*8+rsp],rdi ; tp[j-1] mov r13,rdx cmp r15,r9 jb NEAR $L$1st4x - mul rbx + mul rbx ; ap[j]*bp[0] add r10,rax mov rax,QWORD[((-16))+r15*8+rcx] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+r15*8+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-24))+r15*8+rsp],r13 + mov QWORD[((-24))+r15*8+rsp],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[((-8))+r15*8+rcx] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax - mov rax,QWORD[rsi] + mov rax,QWORD[rsi] ; ap[0] adc rdx,0 - add rdi,r11 + add rdi,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],rdi + mov QWORD[((-16))+r15*8+rsp],rdi ; tp[j-1] mov r13,rdx xor rdi,rdi add r13,r10 adc rdi,0 mov QWORD[((-8))+r15*8+rsp],r13 - mov QWORD[r15*8+rsp],rdi + mov QWORD[r15*8+rsp],rdi ; store upmost overflow bit - lea r14,[1+r14] + lea r14,[1+r14] ; i++ ALIGN 4 $L$outer4x: - mov rbx,QWORD[r14*8+r12] - xor r15,r15 + mov rbx,QWORD[r14*8+r12] ; m0=bp[i] + xor r15,r15 ; j=0 mov r10,QWORD[rsp] mov rbp,r8 - mul rbx - add r10,rax + mul rbx ; ap[0]*bp[i] + add r10,rax ; ap[0]*bp[i]+tp[0] mov rax,QWORD[rcx] adc rdx,0 - imul rbp,r10 + imul rbp,r10 ; tp[0]*n0 mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; "%r13", discarded mov rax,QWORD[8+rsi] adc rdx,0 mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[8+rcx] adc rdx,0 - add r11,QWORD[8+rsp] + add r11,QWORD[8+rsp] ; +tp[1] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[16+rsi] adc rdx,0 - add rdi,r11 - lea r15,[4+r15] + add rdi,r11 ; np[j]*m1+ap[j]*bp[i]+tp[j] + lea r15,[4+r15] ; j+=2 adc rdx,0 - mov QWORD[rsp],rdi + mov QWORD[rsp],rdi ; tp[j-1] mov r13,rdx jmp NEAR $L$inner4x ALIGN 16 $L$inner4x: - mul rbx + mul rbx ; ap[j]*bp[i] add r10,rax mov rax,QWORD[((-16))+r15*8+rcx] adc rdx,0 - add r10,QWORD[((-16))+r15*8+rsp] + add r10,QWORD[((-16))+r15*8+rsp] ; ap[j]*bp[i]+tp[j] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+r15*8+rsi] adc rdx,0 add r13,r10 adc rdx,0 - mov QWORD[((-24))+r15*8+rsp],r13 + mov QWORD[((-24))+r15*8+rsp],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[((-8))+r15*8+rcx] adc rdx,0 @@ -533,148 +533,148 @@ adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[r15*8+rsi] adc rdx,0 add rdi,r11 adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],rdi + mov QWORD[((-16))+r15*8+rsp],rdi ; tp[j-1] mov r13,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r10,rax mov rax,QWORD[r15*8+rcx] adc rdx,0 - add r10,QWORD[r15*8+rsp] + add r10,QWORD[r15*8+rsp] ; ap[j]*bp[i]+tp[j] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[8+r15*8+rsi] adc rdx,0 add r13,r10 adc rdx,0 - mov QWORD[((-8))+r15*8+rsp],r13 + mov QWORD[((-8))+r15*8+rsp],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[8+r15*8+rcx] adc rdx,0 add r11,QWORD[8+r15*8+rsp] adc rdx,0 - lea r15,[4+r15] + lea r15,[4+r15] ; j++ mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[((-16))+r15*8+rsi] adc rdx,0 add rdi,r11 adc rdx,0 - mov QWORD[((-32))+r15*8+rsp],rdi + mov QWORD[((-32))+r15*8+rsp],rdi ; tp[j-1] mov r13,rdx cmp r15,r9 jb NEAR $L$inner4x - mul rbx + mul rbx ; ap[j]*bp[i] add r10,rax mov rax,QWORD[((-16))+r15*8+rcx] adc rdx,0 - add r10,QWORD[((-16))+r15*8+rsp] + add r10,QWORD[((-16))+r15*8+rsp] ; ap[j]*bp[i]+tp[j] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+r15*8+rsi] adc rdx,0 add r13,r10 adc rdx,0 - mov QWORD[((-24))+r15*8+rsp],r13 + mov QWORD[((-24))+r15*8+rsp],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[((-8))+r15*8+rcx] adc rdx,0 add r11,QWORD[((-8))+r15*8+rsp] adc rdx,0 - lea r14,[1+r14] + lea r14,[1+r14] ; i++ mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax - mov rax,QWORD[rsi] + mov rax,QWORD[rsi] ; ap[0] adc rdx,0 add rdi,r11 adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],rdi + mov QWORD[((-16))+r15*8+rsp],rdi ; tp[j-1] mov r13,rdx xor rdi,rdi add r13,r10 adc rdi,0 - add r13,QWORD[r9*8+rsp] + add r13,QWORD[r9*8+rsp] ; pull upmost overflow bit adc rdi,0 mov QWORD[((-8))+r15*8+rsp],r13 - mov QWORD[r15*8+rsp],rdi + mov QWORD[r15*8+rsp],rdi ; store upmost overflow bit cmp r14,r9 jb NEAR $L$outer4x - mov rdi,QWORD[16+r9*8+rsp] + mov rdi,QWORD[16+r9*8+rsp] ; restore %rdi lea r15,[((-4))+r9] - mov rax,QWORD[rsp] - mov rdx,QWORD[8+rsp] - shr r15,2 - lea rsi,[rsp] - xor r14,r14 + mov rax,QWORD[rsp] ; tp[0] + mov rdx,QWORD[8+rsp] ; tp[1] + shr r15,2 ; j=num/4-1 + lea rsi,[rsp] ; borrow ap for tp + xor r14,r14 ; i=0 and clear CF! sub rax,QWORD[rcx] - mov rbx,QWORD[16+rsi] - mov rbp,QWORD[24+rsi] + mov rbx,QWORD[16+rsi] ; tp[2] + mov rbp,QWORD[24+rsi] ; tp[3] sbb rdx,QWORD[8+rcx] $L$sub4x: - mov QWORD[r14*8+rdi],rax - mov QWORD[8+r14*8+rdi],rdx + mov QWORD[r14*8+rdi],rax ; rp[i]=tp[i]-np[i] + mov QWORD[8+r14*8+rdi],rdx ; rp[i]=tp[i]-np[i] sbb rbx,QWORD[16+r14*8+rcx] - mov rax,QWORD[32+r14*8+rsi] + mov rax,QWORD[32+r14*8+rsi] ; tp[i+1] mov rdx,QWORD[40+r14*8+rsi] sbb rbp,QWORD[24+r14*8+rcx] - mov QWORD[16+r14*8+rdi],rbx - mov QWORD[24+r14*8+rdi],rbp + mov QWORD[16+r14*8+rdi],rbx ; rp[i]=tp[i]-np[i] + mov QWORD[24+r14*8+rdi],rbp ; rp[i]=tp[i]-np[i] sbb rax,QWORD[32+r14*8+rcx] mov rbx,QWORD[48+r14*8+rsi] mov rbp,QWORD[56+r14*8+rsi] sbb rdx,QWORD[40+r14*8+rcx] - lea r14,[4+r14] - dec r15 + lea r14,[4+r14] ; i++ + dec r15 ; doesn't affect CF! jnz NEAR $L$sub4x - mov QWORD[r14*8+rdi],rax - mov rax,QWORD[32+r14*8+rsi] + mov QWORD[r14*8+rdi],rax ; rp[i]=tp[i]-np[i] + mov rax,QWORD[32+r14*8+rsi] ; load overflow bit sbb rbx,QWORD[16+r14*8+rcx] - mov QWORD[8+r14*8+rdi],rdx + mov QWORD[8+r14*8+rdi],rdx ; rp[i]=tp[i]-np[i] sbb rbp,QWORD[24+r14*8+rcx] - mov QWORD[16+r14*8+rdi],rbx + mov QWORD[16+r14*8+rdi],rbx ; rp[i]=tp[i]-np[i] - sbb rax,0 - mov QWORD[24+r14*8+rdi],rbp + sbb rax,0 ; handle upmost overflow bit + mov QWORD[24+r14*8+rdi],rbp ; rp[i]=tp[i]-np[i] pxor xmm0,xmm0 movq xmm4,rax pcmpeqd xmm5,xmm5 pshufd xmm4,xmm4,0 mov r15,r9 pxor xmm5,xmm4 - shr r15,2 - xor eax,eax + shr r15,2 ; j=num/4 + xor eax,eax ; i=0 jmp NEAR $L$copy4x ALIGN 16 -$L$copy4x: +$L$copy4x: ; conditional copy movdqa xmm1,XMMWORD[rax*1+rsp] movdqu xmm2,XMMWORD[rax*1+rdi] pand xmm1,xmm4 @@ -692,9 +692,9 @@ lea rax,[32+rax] dec r15 jnz NEAR $L$copy4x - mov rsi,QWORD[8+r9*8+rsp] + mov rsi,QWORD[8+r9*8+rsp] ; restore %rsp - +; No return value mov r15,QWORD[((-48))+rsi] mov r14,QWORD[((-40))+rsi] @@ -715,8 +715,8 @@ ret $L$SEH_end_bn_mul4x_mont: -EXTERN bn_sqrx8x_internal -EXTERN bn_sqr8x_internal +EXTERN bn_sqrx8x_internal ; see x86_64-mont5 module +EXTERN bn_sqr8x_internal ; see x86_64-mont5 module global bn_sqr8x_mont @@ -754,30 +754,30 @@ $L$sqr8x_prologue: mov r10d,r9d - shl r9d,3 - shl r10,3+2 + shl r9d,3 ; convert %r9 to bytes + shl r10,3+2 ; 4*%r9 neg r9 - - - - - +; ############################################################# +; ensure that stack frame doesn't alias with %rsi modulo +; 4096. this is done to allow memory disambiguation logic +; do its job. +; lea r11,[((-64))+r9*2+rsp] mov rbp,rsp - mov r8,QWORD[r8] + mov r8,QWORD[r8] ; *n0 sub r11,rsi and r11,4095 cmp r10,r11 jb NEAR $L$sqr8x_sp_alt - sub rbp,r11 - lea rbp,[((-64))+r9*2+rbp] + sub rbp,r11 ; align with %rsi + lea rbp,[((-64))+r9*2+rbp] ; future alloca(frame+2*%r9) jmp NEAR $L$sqr8x_sp_done ALIGN 32 $L$sqr8x_sp_alt: - lea r10,[((4096-64))+r9*2] - lea rbp,[((-64))+r9*2+rbp] + lea r10,[((4096-64))+r9*2] ; 4096-frame-2*%r9 + lea rbp,[((-64))+r9*2+rbp] ; future alloca(frame+2*%r9) sub r11,r10 mov r10,0 cmovc r11,r10 @@ -805,41 +805,41 @@ neg r9 mov QWORD[32+rsp],r8 - mov QWORD[40+rsp],rax + mov QWORD[40+rsp],rax ; save original %rsp $L$sqr8x_body: - movq xmm2,rcx + movq xmm2,rcx ; save pointer to modulus pxor xmm0,xmm0 - movq xmm1,rdi - movq xmm3,r10 + movq xmm1,rdi ; save %rdi + movq xmm3,r10 ; -%r9 test rdx,rdx jz NEAR $L$sqr8x_nox - call bn_sqrx8x_internal - - - - + call bn_sqrx8x_internal ; see x86_64-mont5 module +; %rax top-most carry +; %rbp nptr +; %rcx -8*num +; %r8 end of tp[2*num] lea rbx,[rcx*1+r8] mov r9,rcx mov rdx,rcx movq rdi,xmm1 - sar rcx,3+2 + sar rcx,3+2 ; %cf=0 jmp NEAR $L$sqr8x_sub ALIGN 32 $L$sqr8x_nox: - call bn_sqr8x_internal - - - - + call bn_sqr8x_internal ; see x86_64-mont5 module +; %rax top-most carry +; %rbp nptr +; %r8 -8*num +; %rdi end of tp[2*num] lea rbx,[r9*1+rdi] mov rcx,r9 mov rdx,r9 movq rdi,xmm1 - sar rcx,3+2 + sar rcx,3+2 ; %cf=0 jmp NEAR $L$sqr8x_sub ALIGN 32 @@ -859,17 +859,17 @@ mov QWORD[16+rdi],r14 mov QWORD[24+rdi],r15 lea rdi,[32+rdi] - inc rcx + inc rcx ; preserves %cf jnz NEAR $L$sqr8x_sub - sbb rax,0 - lea rbx,[r9*1+rbx] - lea rdi,[r9*1+rdi] + sbb rax,0 ; top-most carry + lea rbx,[r9*1+rbx] ; rewind + lea rdi,[r9*1+rdi] ; rewind movq xmm1,rax pxor xmm0,xmm0 pshufd xmm1,xmm1,0 - mov rsi,QWORD[40+rsp] + mov rsi,QWORD[40+rsp] ; restore %rsp jmp NEAR $L$sqr8x_cond_copy @@ -881,7 +881,7 @@ movdqu xmm4,XMMWORD[rdi] movdqu xmm5,XMMWORD[16+rdi] lea rdi,[32+rdi] - movdqa XMMWORD[(-32)+rbx],xmm0 + movdqa XMMWORD[(-32)+rbx],xmm0 ; zero tp movdqa XMMWORD[(-16)+rbx],xmm0 movdqa XMMWORD[(-32)+rdx*1+rbx],xmm0 movdqa XMMWORD[(-16)+rdx*1+rbx],xmm0 @@ -898,7 +898,7 @@ add r9,32 jnz NEAR $L$sqr8x_cond_copy - +; No return value mov r15,QWORD[((-48))+rsi] mov r14,QWORD[((-40))+rsi] @@ -953,11 +953,11 @@ $L$mulx4x_prologue: - shl r9d,3 + shl r9d,3 ; convert %r9 to bytes xor r10,r10 - sub r10,r9 - mov r8,QWORD[r8] - lea rbp,[((-72))+r10*1+rsp] + sub r10,r9 ; -%r9 + mov r8,QWORD[r8] ; *n0 + lea rbp,[((-72))+r10*1+rsp] ; future alloca(frame+%r9+8) and rbp,-128 mov r11,rsp sub r11,rbp @@ -977,62 +977,62 @@ $L$mulx4x_page_walk_done: lea r10,[r9*1+rdx] - - - - - - - - - - - - - mov QWORD[rsp],r9 +; ############################################################# +; Stack layout +; +0 num +; +8 off-loaded &b[i] +; +16 end of b[num] +; +24 saved n0 +; +32 saved rp +; +40 saved %rsp +; +48 inner counter +; +56 +; +64 tmp[num+1] +; + mov QWORD[rsp],r9 ; save %r9 shr r9,5 - mov QWORD[16+rsp],r10 + mov QWORD[16+rsp],r10 ; end of b[num] sub r9,1 - mov QWORD[24+rsp],r8 - mov QWORD[32+rsp],rdi - mov QWORD[40+rsp],rax + mov QWORD[24+rsp],r8 ; save *n0 + mov QWORD[32+rsp],rdi ; save %rdi + mov QWORD[40+rsp],rax ; save original %rsp - mov QWORD[48+rsp],r9 + mov QWORD[48+rsp],r9 ; inner counter jmp NEAR $L$mulx4x_body ALIGN 32 $L$mulx4x_body: lea rdi,[8+rdx] - mov rdx,QWORD[rdx] + mov rdx,QWORD[rdx] ; b[0], %rdx==%rdx actually lea rbx,[((64+32))+rsp] mov r9,rdx - mulx rax,r8,QWORD[rsi] - mulx r14,r11,QWORD[8+rsi] + mulx rax,r8,QWORD[rsi] ; a[0]*b[0] + mulx r14,r11,QWORD[8+rsi] ; a[1]*b[0] add r11,rax - mov QWORD[8+rsp],rdi - mulx r13,r12,QWORD[16+rsi] + mov QWORD[8+rsp],rdi ; off-load &b[i] + mulx r13,r12,QWORD[16+rsi] ; ... adc r12,r14 adc r13,0 - mov rdi,r8 - imul r8,QWORD[24+rsp] - xor rbp,rbp + mov rdi,r8 ; borrow %rdi + imul r8,QWORD[24+rsp] ; "t[0]"*n0 + xor rbp,rbp ; cf=0, of=0 mulx r14,rax,QWORD[24+rsi] mov rdx,r8 lea rsi,[32+rsi] adcx r13,rax - adcx r14,rbp + adcx r14,rbp ; cf=0 mulx r10,rax,QWORD[rcx] - adcx rdi,rax + adcx rdi,rax ; discarded adox r10,r11 mulx r11,rax,QWORD[8+rcx] adcx r10,rax adox r11,r12 - DB 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 - mov rdi,QWORD[48+rsp] + DB 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 ; mulx 2*8(%rcx),%rax,%r12 + mov rdi,QWORD[48+rsp] ; counter value mov QWORD[((-32))+rbx],r10 adcx r11,rax adox r12,r13 @@ -1040,7 +1040,7 @@ mov rdx,r9 mov QWORD[((-24))+rbx],r11 adcx r12,rax - adox r15,rbp + adox r15,rbp ; of=0 lea rcx,[32+rcx] mov QWORD[((-16))+rbx],r12 @@ -1048,18 +1048,18 @@ ALIGN 32 $L$mulx4x_1st: - adcx r15,rbp - mulx rax,r10,QWORD[rsi] + adcx r15,rbp ; cf=0, modulo-scheduled + mulx rax,r10,QWORD[rsi] ; a[4]*b[0] adcx r10,r14 - mulx r14,r11,QWORD[8+rsi] + mulx r14,r11,QWORD[8+rsi] ; a[5]*b[0] adcx r11,rax - mulx rax,r12,QWORD[16+rsi] + mulx rax,r12,QWORD[16+rsi] ; ... adcx r12,r14 mulx r14,r13,QWORD[24+rsi] DB 0x67,0x67 mov rdx,r8 adcx r13,rax - adcx r14,rbp + adcx r14,rbp ; cf=0 lea rsi,[32+rsi] lea rbx,[32+rbx] @@ -1083,43 +1083,43 @@ lea rcx,[32+rcx] mov QWORD[((-16))+rbx],r13 - dec rdi + dec rdi ; of=0, pass cf jnz NEAR $L$mulx4x_1st - mov rax,QWORD[rsp] - mov rdi,QWORD[8+rsp] - adc r15,rbp + mov rax,QWORD[rsp] ; load num + mov rdi,QWORD[8+rsp] ; re-load &b[i] + adc r15,rbp ; modulo-scheduled add r14,r15 - sbb r15,r15 + sbb r15,r15 ; top-most carry mov QWORD[((-8))+rbx],r14 jmp NEAR $L$mulx4x_outer ALIGN 32 $L$mulx4x_outer: - mov rdx,QWORD[rdi] - lea rdi,[8+rdi] - sub rsi,rax - mov QWORD[rbx],r15 + mov rdx,QWORD[rdi] ; b[i] + lea rdi,[8+rdi] ; b++ + sub rsi,rax ; rewind %rsi + mov QWORD[rbx],r15 ; save top-most carry lea rbx,[((64+32))+rsp] - sub rcx,rax + sub rcx,rax ; rewind %rcx - mulx r11,r8,QWORD[rsi] - xor ebp,ebp + mulx r11,r8,QWORD[rsi] ; a[0]*b[i] + xor ebp,ebp ; xor %rbp,%rbp # cf=0, of=0 mov r9,rdx - mulx r12,r14,QWORD[8+rsi] + mulx r12,r14,QWORD[8+rsi] ; a[1]*b[i] adox r8,QWORD[((-32))+rbx] adcx r11,r14 - mulx r13,r15,QWORD[16+rsi] + mulx r13,r15,QWORD[16+rsi] ; ... adox r11,QWORD[((-24))+rbx] adcx r12,r15 adox r12,QWORD[((-16))+rbx] adcx r13,rbp adox r13,rbp - mov QWORD[8+rsp],rdi + mov QWORD[8+rsp],rdi ; off-load &b[i] mov r15,r8 - imul r8,QWORD[24+rsp] - xor ebp,ebp + imul r8,QWORD[24+rsp] ; "t[0]"*n0 + xor ebp,ebp ; xor %rbp,%rbp # cf=0, of=0 mulx r14,rax,QWORD[24+rsi] mov rdx,r8 @@ -1130,7 +1130,7 @@ adox r14,rbp mulx r10,rax,QWORD[rcx] - adcx r15,rax + adcx r15,rax ; discarded adox r10,r11 mulx r11,rax,QWORD[8+rcx] adcx r10,rax @@ -1144,21 +1144,21 @@ mov QWORD[((-24))+rbx],r11 lea rcx,[32+rcx] adcx r12,rax - adox r15,rbp - mov rdi,QWORD[48+rsp] + adox r15,rbp ; of=0 + mov rdi,QWORD[48+rsp] ; counter value mov QWORD[((-16))+rbx],r12 jmp NEAR $L$mulx4x_inner ALIGN 32 $L$mulx4x_inner: - mulx rax,r10,QWORD[rsi] - adcx r15,rbp + mulx rax,r10,QWORD[rsi] ; a[4]*b[i] + adcx r15,rbp ; cf=0, modulo-scheduled adox r10,r14 - mulx r14,r11,QWORD[8+rsi] + mulx r14,r11,QWORD[8+rsi] ; a[5]*b[i] adcx r10,QWORD[rbx] adox r11,rax - mulx rax,r12,QWORD[16+rsi] + mulx rax,r12,QWORD[16+rsi] ; ... adcx r11,QWORD[8+rbx] adox r12,r14 mulx r14,r13,QWORD[24+rsi] @@ -1166,10 +1166,10 @@ adcx r12,QWORD[16+rbx] adox r13,rax adcx r13,QWORD[24+rbx] - adox r14,rbp + adox r14,rbp ; of=0 lea rsi,[32+rsi] lea rbx,[32+rbx] - adcx r14,rbp + adcx r14,rbp ; cf=0 adox r10,r15 mulx r15,rax,QWORD[rcx] @@ -1191,26 +1191,26 @@ lea rcx,[32+rcx] mov QWORD[((-16))+rbx],r13 - dec rdi + dec rdi ; of=0, pass cf jnz NEAR $L$mulx4x_inner - mov rax,QWORD[rsp] - mov rdi,QWORD[8+rsp] - adc r15,rbp - sub rbp,QWORD[rbx] + mov rax,QWORD[rsp] ; load num + mov rdi,QWORD[8+rsp] ; re-load &b[i] + adc r15,rbp ; modulo-scheduled + sub rbp,QWORD[rbx] ; pull top-most carry adc r14,r15 - sbb r15,r15 + sbb r15,r15 ; top-most carry mov QWORD[((-8))+rbx],r14 cmp rdi,QWORD[16+rsp] jne NEAR $L$mulx4x_outer lea rbx,[64+rsp] - sub rcx,rax + sub rcx,rax ; rewind %rcx neg r15 mov rdx,rax - shr rax,3+2 - mov rdi,QWORD[32+rsp] + shr rax,3+2 ; %cf=0 + mov rdi,QWORD[32+rsp] ; restore rp jmp NEAR $L$mulx4x_sub ALIGN 32 @@ -1230,17 +1230,17 @@ mov QWORD[16+rdi],r13 mov QWORD[24+rdi],r14 lea rdi,[32+rdi] - dec rax + dec rax ; preserves %cf jnz NEAR $L$mulx4x_sub - sbb r15,0 + sbb r15,0 ; top-most carry lea rbx,[64+rsp] - sub rdi,rdx + sub rdi,rdx ; rewind movq xmm1,r15 pxor xmm0,xmm0 pshufd xmm1,xmm1,0 - mov rsi,QWORD[40+rsp] + mov rsi,QWORD[40+rsp] ; restore %rsp jmp NEAR $L$mulx4x_cond_copy @@ -1252,7 +1252,7 @@ movdqu xmm4,XMMWORD[rdi] movdqu xmm5,XMMWORD[16+rdi] lea rdi,[32+rdi] - movdqa XMMWORD[(-32)+rbx],xmm0 + movdqa XMMWORD[(-32)+rbx],xmm0 ; zero tp movdqa XMMWORD[(-16)+rbx],xmm0 pcmpeqd xmm0,xmm1 pand xmm2,xmm1 @@ -1269,7 +1269,7 @@ mov QWORD[rbx],rdx - +; No return value mov r15,QWORD[((-48))+rsi] mov r14,QWORD[((-40))+rsi] @@ -1311,26 +1311,26 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; end of prologue label + cmp rbx,r10 ; context->Rip<end of prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail - mov r10,QWORD[192+r8] - mov rax,QWORD[8+r10*8+rax] + mov r10,QWORD[192+r8] ; pull %r9 + mov rax,QWORD[8+r10*8+rax] ; pull saved stack pointer jmp NEAR $L$common_pop_regs @@ -1349,30 +1349,30 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; end of prologue label + cmp rbx,r10 ; context->Rip<.Lsqr_prologue jb NEAR $L$common_seh_tail - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; body label + cmp rbx,r10 ; context->Rip<.Lsqr_body jb NEAR $L$common_pop_regs - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[8+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[8+r11] ; HandlerData[2] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=.Lsqr_epilogue jae NEAR $L$common_seh_tail - mov rax,QWORD[40+rax] + mov rax,QWORD[40+rax] ; pull saved stack pointer $L$common_pop_regs: mov rbx,QWORD[((-8))+rax] @@ -1381,40 +1381,40 @@ mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R15 $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -1449,20 +1449,20 @@ $L$SEH_info_bn_mul_mont_nohw: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase + DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_bn_mul4x_mont: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase + DD $L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase ; HandlerData[] $L$SEH_info_bn_sqr8x_mont: DB 9,0,0,0 DD sqr_handler wrt ..imagebase - DD $L$sqr8x_prologue wrt ..imagebase,$L$sqr8x_body wrt ..imagebase,$L$sqr8x_epilogue wrt ..imagebase + DD $L$sqr8x_prologue wrt ..imagebase,$L$sqr8x_body wrt ..imagebase,$L$sqr8x_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 $L$SEH_info_bn_mulx4x_mont: DB 9,0,0,0 DD sqr_handler wrt ..imagebase - DD $L$mulx4x_prologue wrt ..imagebase,$L$mulx4x_body wrt ..imagebase,$L$mulx4x_epilogue wrt ..imagebase + DD $L$mulx4x_prologue wrt ..imagebase,$L$mulx4x_body wrt ..imagebase,$L$mulx4x_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
diff --git a/gen/bcm/x86_64-mont5-apple.S b/gen/bcm/x86_64-mont5-apple.S index 102f0ba..ae91a95 100644 --- a/gen/bcm/x86_64-mont5-apple.S +++ b/gen/bcm/x86_64-mont5-apple.S
@@ -13,12 +13,12 @@ _bn_mul_mont_gather5_nohw: _CET_ENDBR - - +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. Zero the upper half to normalize it. movl %r9d,%r9d movq %rsp,%rax - movd 8(%rsp),%xmm5 + movd 8(%rsp),%xmm5 // load 7th argument pushq %rbx pushq %rbp @@ -34,18 +34,18 @@ negq %r9 movq %rsp,%r11 - leaq -280(%rsp,%r9,8),%r10 - negq %r9 - andq $-1024,%r10 + leaq -280(%rsp,%r9,8),%r10 // future alloca(8*(num+2)+256+8) + negq %r9 // restore %r9 + andq $-1024,%r10 // minimize TLB usage - - - - - - - - +// An OS-agnostic version of __chkstk. +// +// Some OSes (Windows) insist on stack being "wired" to +// physical memory in strictly sequential manner, i.e. if stack +// allocation spans two pages, then reference to farmost one can +// be punishable by SEGV. But page walking can do good even on +// other OSes, because it guarantees that villain thread hits +// the guard page before it can make damage to innocent one... subq %r10,%r11 andq $-4096,%r11 leaq (%r10,%r11,1),%rsp @@ -62,35 +62,35 @@ L$mul_page_walk_done: leaq L$inc(%rip),%r10 - movq %rax,8(%rsp,%r9,8) + movq %rax,8(%rsp,%r9,8) // tp[num+1]=%rsp L$mul_body: - leaq 128(%rdx),%r12 - movdqa 0(%r10),%xmm0 - movdqa 16(%r10),%xmm1 - leaq 24-112(%rsp,%r9,8),%r10 + leaq 128(%rdx),%r12 // reassign %rdx (+size optimization) + movdqa 0(%r10),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%r10),%xmm1 // 00000002000000020000000200000002 + leaq 24-112(%rsp,%r9,8),%r10 // place the mask after tp[num+3] (+ICache optimization) andq $-16,%r10 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast index movdqa %xmm1,%xmm4 movdqa %xmm1,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 .byte 0x67 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,112(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,128(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,144(%r10) movdqa %xmm4,%xmm2 @@ -99,17 +99,17 @@ movdqa %xmm3,160(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,176(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,192(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,208(%r10) movdqa %xmm4,%xmm2 @@ -118,17 +118,17 @@ movdqa %xmm3,224(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,240(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,256(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,272(%r10) movdqa %xmm4,%xmm2 @@ -147,7 +147,7 @@ pcmpeqd %xmm5,%xmm3 movdqa %xmm2,336(%r10) - pand 64(%r12),%xmm0 + pand 64(%r12),%xmm0 // while it's still in register pand 80(%r12),%xmm1 pand 96(%r12),%xmm2 @@ -192,33 +192,33 @@ por %xmm2,%xmm0 por %xmm3,%xmm1 por %xmm1,%xmm0 - - pshufd $0x4e,%xmm0,%xmm1 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm0,%xmm1 // Swap upper and lower halves. por %xmm1,%xmm0 leaq 256(%r12),%r12 - movq %xmm0,%rbx + movq %xmm0,%rbx // m0=bp[0] - movq (%r8),%r8 + movq (%r8),%r8 // pull n0[0] value movq (%rsi),%rax - xorq %r14,%r14 - xorq %r15,%r15 + xorq %r14,%r14 // i=0 + xorq %r15,%r15 // j=0 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp L$1st_enter .p2align 4 @@ -226,30 +226,30 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] movq %r10,%r11 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 L$1st_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 - jne L$1st - + jne L$1st // note that upon exit %r15==%r9, so +// they can be used interchangeably addq %rax,%r13 adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-16(%rsp,%r9,8) + movq %r13,-16(%rsp,%r9,8) // tp[num-1] movq %rdx,%r13 movq %r10,%r11 @@ -257,13 +257,13 @@ addq %r11,%r13 adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ jmp L$outer .p2align 4 L$outer: - leaq 24+128(%rsp,%r9,8),%rdx + leaq 24+128(%rsp,%r9,8),%rdx // where 256-byte mask is (+size optimization) andq $-16,%rdx pxor %xmm4,%xmm4 pxor %xmm5,%xmm5 @@ -316,34 +316,34 @@ por %xmm2,%xmm4 por %xmm3,%xmm5 por %xmm5,%xmm4 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 leaq 256(%r12),%r12 - movq (%rsi),%rax - movq %xmm0,%rbx + movq (%rsi),%rax // ap[0] + movq %xmm0,%rbx // m0=bp[i] - xorq %r15,%r15 + xorq %r15,%r15 // j=0 movq %r8,%rbp movq (%rsp),%r10 - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx - movq 8(%rsp),%r10 + movq 8(%rsp),%r10 // tp[1] movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp L$inner_enter .p2align 4 @@ -351,78 +351,78 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r15,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 L$inner_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - addq %r11,%r10 + addq %r11,%r10 // ap[j]*bp[i]+tp[j] movq %rdx,%r11 adcq $0,%r11 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 - jne L$inner - + jne L$inner // note that upon exit %r15==%r9, so +// they can be used interchangeably addq %rax,%r13 adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r9,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r9,8) + movq %r13,-16(%rsp,%r9,8) // tp[num-1] movq %rdx,%r13 xorq %rdx,%rdx addq %r11,%r13 adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // pull upmost overflow bit adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ cmpq %r9,%r14 jb L$outer - xorq %r14,%r14 - movq (%rsp),%rax - leaq (%rsp),%rsi - movq %r9,%r15 + xorq %r14,%r14 // i=0 and clear CF + movq (%rsp),%rax // tp[0] + leaq (%rsp),%rsi // borrow ap for tp + movq %r9,%r15 // j=num jmp L$sub .p2align 4 L$sub: sbbq (%rcx,%r14,8),%rax - movq %rax,(%rdi,%r14,8) - movq 8(%rsi,%r14,8),%rax - leaq 1(%r14),%r14 - decq %r15 + movq %rax,(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq 8(%rsi,%r14,8),%rax // tp[i+1] + leaq 1(%r14),%r14 // i++ + decq %r15 // doesn't affect CF jnz L$sub - sbbq $0,%rax + sbbq $0,%rax // handle upmost overflow bit movq $-1,%rbx xorq %rax,%rbx xorq %r14,%r14 - movq %r9,%r15 + movq %r9,%r15 // j=num -L$copy: +L$copy: // conditional copy movq (%rdi,%r14,8),%rcx movq (%rsp,%r14,8),%rdx andq %rbx,%rcx andq %rax,%rdx - movq %r14,(%rsp,%r14,8) + movq %r14,(%rsp,%r14,8) // zap temporary vector orq %rcx,%rdx - movq %rdx,(%rdi,%r14,8) + movq %rdx,(%rdi,%r14,8) // rp[i]=tp[i] leaq 1(%r14),%r14 subq $1,%r15 jnz L$copy - movq 8(%rsp,%r9,8),%rsi + movq 8(%rsp,%r9,8),%rsi // restore %rsp movq $1,%rax @@ -470,36 +470,36 @@ L$mul4x_prologue: .byte 0x67 +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, predates the first access. + shll $3,%r9d // convert %r9 to bytes + leaq (%r9,%r9,2),%r10 // 3*%r9 in bytes + negq %r9 // -%r9 - - - shll $3,%r9d - leaq (%r9,%r9,2),%r10 - negq %r9 - - - - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with +3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra [num] is allocated in order +// to align with bn_power5's frame, which is cleansed after +// completing exponentiation. Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb L$mul4xsp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rdi + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) jmp L$mul4xsp_done .p2align 5 L$mul4xsp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -530,7 +530,7 @@ call mul4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp movq $1,%rax @@ -558,36 +558,36 @@ .alt_entry mul4x_internal mul4x_internal: - shlq $5,%r9 - movd 8(%rax),%xmm5 + shlq $5,%r9 // %r9 was in bytes + movd 8(%rax),%xmm5 // load 7th argument, index leaq L$inc(%rip),%rax - leaq 128(%rdx,%r9,1),%r13 - shrq $5,%r9 - movdqa 0(%rax),%xmm0 - movdqa 16(%rax),%xmm1 - leaq 88-112(%rsp,%r9,1),%r10 - leaq 128(%rdx),%r12 + leaq 128(%rdx,%r9,1),%r13 // end of powers table (+size optimization) + shrq $5,%r9 // restore %r9 + movdqa 0(%rax),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%rax),%xmm1 // 00000002000000020000000200000002 + leaq 88-112(%rsp,%r9,1),%r10 // place the mask after tp[num+1] (+ICache optimization) + leaq 128(%rdx),%r12 // size optimization - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast index movdqa %xmm1,%xmm4 .byte 0x67,0x67 movdqa %xmm1,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 .byte 0x67 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,112(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,128(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,144(%r10) movdqa %xmm4,%xmm2 @@ -596,17 +596,17 @@ movdqa %xmm3,160(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,176(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,192(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,208(%r10) movdqa %xmm4,%xmm2 @@ -615,17 +615,17 @@ movdqa %xmm3,224(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,240(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,256(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,272(%r10) movdqa %xmm4,%xmm2 @@ -644,7 +644,7 @@ pcmpeqd %xmm5,%xmm3 movdqa %xmm2,336(%r10) - pand 64(%r12),%xmm0 + pand 64(%r12),%xmm0 // while it's still in register pand 80(%r12),%xmm1 pand 96(%r12),%xmm2 @@ -689,31 +689,31 @@ por %xmm2,%xmm0 por %xmm3,%xmm1 por %xmm1,%xmm0 - - pshufd $0x4e,%xmm0,%xmm1 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm0,%xmm1 // Swap upper and lower halves. por %xmm1,%xmm0 leaq 256(%r12),%r12 - movq %xmm0,%rbx + movq %xmm0,%rbx // m0=bp[0] - movq %r13,16+8(%rsp) - movq %rdi,56+8(%rsp) + movq %r13,16+8(%rsp) // save end of b[num] + movq %rdi,56+8(%rsp) // save %rdi - movq (%r8),%r8 + movq (%r8),%r8 // pull n0[0] value movq (%rsi),%rax - leaq (%rsi,%r9,1),%rsi + leaq (%rsi,%r9,1),%rsi // end of a[num] negq %r9 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 leaq 64+8(%rsp),%r14 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi,%r9,1),%rax adcq $0,%rdx movq %rdx,%rdi @@ -729,7 +729,7 @@ movq 16(%rsi,%r9,1),%rax adcq $0,%rdx addq %r11,%rdi - leaq 32(%r9),%r15 + leaq 32(%r9),%r15 // j=4 leaq 32(%rcx),%rcx adcq $0,%rdx movq %rdi,(%r14) @@ -738,103 +738,103 @@ .p2align 5 L$1st4x: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx),%rax leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,1),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,1),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%r14) + movq %rdi,-16(%r14) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq 0(%rcx),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,1),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-8(%r14) + movq %r13,-8(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi,%r15,1),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] leaq 32(%rcx),%rcx adcq $0,%rdx - movq %rdi,(%r14) + movq %rdi,(%r14) // tp[j-1] movq %rdx,%r13 - addq $32,%r15 + addq $32,%r15 // j+=4 jnz L$1st4x - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx),%rax leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi,%r9,1),%rax + movq (%rsi,%r9,1),%rax // ap[0] adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%r14) + movq %rdi,-16(%r14) // tp[j-1] movq %rdx,%r13 - leaq (%rcx,%r9,1),%rcx + leaq (%rcx,%r9,1),%rcx // rewind %rcx xorq %rdi,%rdi addq %r10,%r13 @@ -845,7 +845,7 @@ .p2align 5 L$outer4x: - leaq 16+128(%r14),%rdx + leaq 16+128(%r14),%rdx // where 256-byte mask is (+size optimization) pxor %xmm4,%xmm4 pxor %xmm5,%xmm5 movdqa -128(%r12),%xmm0 @@ -897,45 +897,45 @@ por %xmm2,%xmm4 por %xmm3,%xmm5 por %xmm5,%xmm4 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 leaq 256(%r12),%r12 - movq %xmm0,%rbx + movq %xmm0,%rbx // m0=bp[i] movq (%r14,%r9,1),%r10 movq %r8,%rbp - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - movq %rdi,(%r14) + movq %rdi,(%r14) // store upmost overflow bit - leaq (%r14,%r9,1),%r14 + leaq (%r14,%r9,1),%r14 // rewind %r14 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // "%r13", discarded movq 8(%rsi,%r9,1),%rax adcq $0,%rdx movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx - addq 8(%r14),%r11 + addq 8(%r14),%r11 // +tp[1] adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi,%r9,1),%rax adcq $0,%rdx - addq %r11,%rdi - leaq 32(%r9),%r15 + addq %r11,%rdi // np[j]*m1+ap[j]*bp[i]+tp[j] + leaq 32(%r9),%r15 // j=4 leaq 32(%rcx),%rcx adcq $0,%rdx movq %rdx,%r13 @@ -943,25 +943,25 @@ .p2align 5 L$inner4x: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx),%rax adcq $0,%rdx - addq 16(%r14),%r10 + addq 16(%r14),%r10 // ap[j]*bp[i]+tp[j] leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,1),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %rdi,-32(%r14) + movq %rdi,-32(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq -8(%rcx),%rax adcq $0,%rdx @@ -969,33 +969,33 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,1),%rax adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq 0(%rcx),%rax adcq $0,%rdx - addq (%r14),%r10 + addq (%r14),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,1),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %rdi,-16(%r14) + movq %rdi,-16(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx @@ -1003,38 +1003,38 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi,%r15,1),%rax adcq $0,%rdx addq %r11,%rdi leaq 32(%rcx),%rcx adcq $0,%rdx - movq %r13,-8(%r14) + movq %r13,-8(%r14) // tp[j-1] movq %rdx,%r13 - addq $32,%r15 + addq $32,%r15 // j+=4 jnz L$inner4x - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx),%rax adcq $0,%rdx - addq 16(%r14),%r10 + addq 16(%r14),%r10 // ap[j]*bp[i]+tp[j] leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %rdi,-32(%r14) + movq %rdi,-32(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq %rbp,%rax movq -8(%rcx),%rbp @@ -1043,39 +1043,39 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi,%r9,1),%rax + movq (%rsi,%r9,1),%rax // ap[0] adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%r13 - movq %rdi,-16(%r14) - leaq (%rcx,%r9,1),%rcx + movq %rdi,-16(%r14) // tp[j-1] + leaq (%rcx,%r9,1),%rcx // rewind %rcx xorq %rdi,%rdi addq %r10,%r13 adcq $0,%rdi - addq (%r14),%r13 - adcq $0,%rdi + addq (%r14),%r13 // pull upmost overflow bit + adcq $0,%rdi // upmost overflow bit movq %r13,-8(%r14) cmpq 16+8(%rsp),%r12 jb L$outer4x xorq %rax,%rax - subq %r13,%rbp - adcq %r15,%r15 + subq %r13,%rbp // compare top-most words + adcq %r15,%r15 // %r15 is zero orq %r15,%rdi - subq %rdi,%rax - leaq (%r14,%r9,1),%rbx + subq %rdi,%rax // %rax=-%rdi + leaq (%r14,%r9,1),%rbx // tptr in .sqr4x_sub movq (%rcx),%r12 - leaq (%rcx),%rbp + leaq (%rcx),%rbp // nptr in .sqr4x_sub movq %r9,%rcx sarq $3+2,%rcx - movq 56+8(%rsp),%rdi - decq %r12 + movq 56+8(%rsp),%rdi // rptr in .sqr4x_sub + decq %r12 // so that after 'not' we get -n[0] xorq %r10,%r10 movq 8(%rbp),%r13 movq 16(%rbp),%r14 @@ -1107,35 +1107,35 @@ L$power5_prologue: - - - - shll $3,%r9d - leal (%r9,%r9,2),%r10d +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, come before the first access. + shll $3,%r9d // convert %r9 to bytes + leal (%r9,%r9,2),%r10d // 3*%r9 negq %r9 - movq (%r8),%r8 + movq (%r8),%r8 // *n0 - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with %rdi+3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb L$pwr_sp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rsi + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) jmp L$pwr_sp_done .p2align 5 L$pwr_sp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -1161,22 +1161,22 @@ movq %r9,%r10 negq %r9 - - - - - - - - - +// ############################################################# +// Stack layout +// +// +0 saved %r9, used in reduction section +// +8 &t[2*%r9], used in reduction section +// +32 saved *n0 +// +40 saved %rsp +// +48 t[2*%r9] +// movq %r8,32(%rsp) - movq %rax,40(%rsp) + movq %rax,40(%rsp) // save original %rsp L$power5_body: - movq %rdi,%xmm1 - movq %rcx,%xmm2 - movq %r10,%xmm3 + movq %rdi,%xmm1 // save %rdi, used in sqr8x + movq %rcx,%xmm2 // save %rcx + movq %r10,%xmm3 // -%r9, used in sqr8x movq %rdx,%xmm4 call __bn_sqr8x_internal @@ -1198,7 +1198,7 @@ call mul4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp movq $1,%rax movq -48(%rsi),%r15 @@ -1231,261 +1231,261 @@ __bn_sqr8x_internal: _CET_ENDBR +// ############################################################# +// Squaring part: +// +// a) multiply-n-add everything but a[i]*a[i]; +// b) shift result of a) by 1 to the left and accumulate +// a[i]*a[i] products; +// +// ############################################################# +// a[1]a[0] +// a[2]a[0] +// a[3]a[0] +// a[2]a[1] +// a[4]a[0] +// a[3]a[1] +// a[5]a[0] +// a[4]a[1] +// a[3]a[2] +// a[6]a[0] +// a[5]a[1] +// a[4]a[2] +// a[7]a[0] +// a[6]a[1] +// a[5]a[2] +// a[4]a[3] +// a[7]a[1] +// a[6]a[2] +// a[5]a[3] +// a[7]a[2] +// a[6]a[3] +// a[5]a[4] +// a[7]a[3] +// a[6]a[4] +// a[7]a[4] +// a[6]a[5] +// a[7]a[5] +// a[7]a[6] +// a[1]a[0] +// a[2]a[0] +// a[3]a[0] +// a[4]a[0] +// a[5]a[0] +// a[6]a[0] +// a[7]a[0] +// a[2]a[1] +// a[3]a[1] +// a[4]a[1] +// a[5]a[1] +// a[6]a[1] +// a[7]a[1] +// a[3]a[2] +// a[4]a[2] +// a[5]a[2] +// a[6]a[2] +// a[7]a[2] +// a[4]a[3] +// a[5]a[3] +// a[6]a[3] +// a[7]a[3] +// a[5]a[4] +// a[6]a[4] +// a[7]a[4] +// a[6]a[5] +// a[7]a[5] +// a[7]a[6] +// a[0]a[0] +// a[1]a[1] +// a[2]a[2] +// a[3]a[3] +// a[4]a[4] +// a[5]a[5] +// a[6]a[6] +// a[7]a[7] + leaq 32(%r10),%rbp // %rbp=-(%r9-32) + leaq (%rsi,%r9,1),%rsi // end of a[] buffer, (%rsi,%rbp)=&ap[2] + movq %r9,%rcx // %rcx=%r9 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - leaq 32(%r10),%rbp - leaq (%rsi,%r9,1),%rsi - - movq %r9,%rcx - - - movq -32(%rsi,%rbp,1),%r14 - leaq 48+8(%rsp,%r9,2),%rdi - movq -24(%rsi,%rbp,1),%rax - leaq -32(%rdi,%rbp,1),%rdi - movq -16(%rsi,%rbp,1),%rbx +// comments apply to %r9==8 case + movq -32(%rsi,%rbp,1),%r14 // a[0] + leaq 48+8(%rsp,%r9,2),%rdi // end of tp[] buffer, &tp[2*%r9] + movq -24(%rsi,%rbp,1),%rax // a[1] + leaq -32(%rdi,%rbp,1),%rdi // end of tp[] window, &tp[2*%r9-"%rbp"] + movq -16(%rsi,%rbp,1),%rbx // a[2] movq %rax,%r15 - mulq %r14 - movq %rax,%r10 - movq %rbx,%rax + mulq %r14 // a[1]*a[0] + movq %rax,%r10 // a[1]*a[0] + movq %rbx,%rax // a[2] movq %rdx,%r11 - movq %r10,-24(%rdi,%rbp,1) + movq %r10,-24(%rdi,%rbp,1) // t[1] - mulq %r14 + mulq %r14 // a[2]*a[0] addq %rax,%r11 movq %rbx,%rax adcq $0,%rdx - movq %r11,-16(%rdi,%rbp,1) + movq %r11,-16(%rdi,%rbp,1) // t[2] movq %rdx,%r10 - movq -8(%rsi,%rbp,1),%rbx - mulq %r15 - movq %rax,%r12 + movq -8(%rsi,%rbp,1),%rbx // a[3] + mulq %r15 // a[2]*a[1] + movq %rax,%r12 // a[2]*a[1]+t[3] movq %rbx,%rax movq %rdx,%r13 leaq (%rbp),%rcx - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[3]*a[0] + addq %rax,%r10 // a[3]*a[0]+a[2]*a[1]+t[3] movq %rbx,%rax movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - movq %r10,-8(%rdi,%rcx,1) + movq %r10,-8(%rdi,%rcx,1) // t[3] jmp L$sqr4x_1st .p2align 5 L$sqr4x_1st: - movq (%rsi,%rcx,1),%rbx - mulq %r15 - addq %rax,%r13 + movq (%rsi,%rcx,1),%rbx // a[4] + mulq %r15 // a[3]*a[1] + addq %rax,%r13 // a[3]*a[1]+t[4] movq %rbx,%rax movq %rdx,%r12 adcq $0,%r12 - mulq %r14 - addq %rax,%r11 - movq %rbx,%rax - movq 8(%rsi,%rcx,1),%rbx + mulq %r14 // a[4]*a[0] + addq %rax,%r11 // a[4]*a[0]+a[3]*a[1]+t[4] + movq %rbx,%rax // a[3] + movq 8(%rsi,%rcx,1),%rbx // a[5] movq %rdx,%r10 adcq $0,%r10 addq %r13,%r11 adcq $0,%r10 - mulq %r15 - addq %rax,%r12 + mulq %r15 // a[4]*a[3] + addq %rax,%r12 // a[4]*a[3]+t[5] movq %rbx,%rax - movq %r11,(%rdi,%rcx,1) + movq %r11,(%rdi,%rcx,1) // t[4] movq %rdx,%r13 adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[5]*a[2] + addq %rax,%r10 // a[5]*a[2]+a[4]*a[3]+t[5] movq %rbx,%rax - movq 16(%rsi,%rcx,1),%rbx + movq 16(%rsi,%rcx,1),%rbx // a[6] movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - mulq %r15 - addq %rax,%r13 + mulq %r15 // a[5]*a[3] + addq %rax,%r13 // a[5]*a[3]+t[6] movq %rbx,%rax - movq %r10,8(%rdi,%rcx,1) + movq %r10,8(%rdi,%rcx,1) // t[5] movq %rdx,%r12 adcq $0,%r12 - mulq %r14 - addq %rax,%r11 - movq %rbx,%rax - movq 24(%rsi,%rcx,1),%rbx + mulq %r14 // a[6]*a[2] + addq %rax,%r11 // a[6]*a[2]+a[5]*a[3]+t[6] + movq %rbx,%rax // a[3] + movq 24(%rsi,%rcx,1),%rbx // a[7] movq %rdx,%r10 adcq $0,%r10 addq %r13,%r11 adcq $0,%r10 - mulq %r15 - addq %rax,%r12 + mulq %r15 // a[6]*a[5] + addq %rax,%r12 // a[6]*a[5]+t[7] movq %rbx,%rax - movq %r11,16(%rdi,%rcx,1) + movq %r11,16(%rdi,%rcx,1) // t[6] movq %rdx,%r13 adcq $0,%r13 leaq 32(%rcx),%rcx - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[7]*a[4] + addq %rax,%r10 // a[7]*a[4]+a[6]*a[5]+t[6] movq %rbx,%rax movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - movq %r10,-8(%rdi,%rcx,1) + movq %r10,-8(%rdi,%rcx,1) // t[7] cmpq $0,%rcx jne L$sqr4x_1st - mulq %r15 + mulq %r15 // a[7]*a[5] addq %rax,%r13 leaq 16(%rbp),%rbp adcq $0,%rdx addq %r11,%r13 adcq $0,%rdx - movq %r13,(%rdi) + movq %r13,(%rdi) // t[8] movq %rdx,%r12 - movq %rdx,8(%rdi) + movq %rdx,8(%rdi) // t[9] jmp L$sqr4x_outer .p2align 5 -L$sqr4x_outer: - movq -32(%rsi,%rbp,1),%r14 - leaq 48+8(%rsp,%r9,2),%rdi - movq -24(%rsi,%rbp,1),%rax - leaq -32(%rdi,%rbp,1),%rdi - movq -16(%rsi,%rbp,1),%rbx +L$sqr4x_outer: // comments apply to %r9==6 case + movq -32(%rsi,%rbp,1),%r14 // a[0] + leaq 48+8(%rsp,%r9,2),%rdi // end of tp[] buffer, &tp[2*%r9] + movq -24(%rsi,%rbp,1),%rax // a[1] + leaq -32(%rdi,%rbp,1),%rdi // end of tp[] window, &tp[2*%r9-"%rbp"] + movq -16(%rsi,%rbp,1),%rbx // a[2] movq %rax,%r15 - mulq %r14 - movq -24(%rdi,%rbp,1),%r10 - addq %rax,%r10 - movq %rbx,%rax + mulq %r14 // a[1]*a[0] + movq -24(%rdi,%rbp,1),%r10 // t[1] + addq %rax,%r10 // a[1]*a[0]+t[1] + movq %rbx,%rax // a[2] adcq $0,%rdx - movq %r10,-24(%rdi,%rbp,1) + movq %r10,-24(%rdi,%rbp,1) // t[1] movq %rdx,%r11 - mulq %r14 + mulq %r14 // a[2]*a[0] addq %rax,%r11 movq %rbx,%rax adcq $0,%rdx - addq -16(%rdi,%rbp,1),%r11 + addq -16(%rdi,%rbp,1),%r11 // a[2]*a[0]+t[2] movq %rdx,%r10 adcq $0,%r10 - movq %r11,-16(%rdi,%rbp,1) + movq %r11,-16(%rdi,%rbp,1) // t[2] xorq %r12,%r12 - movq -8(%rsi,%rbp,1),%rbx - mulq %r15 - addq %rax,%r12 + movq -8(%rsi,%rbp,1),%rbx // a[3] + mulq %r15 // a[2]*a[1] + addq %rax,%r12 // a[2]*a[1]+t[3] movq %rbx,%rax adcq $0,%rdx addq -8(%rdi,%rbp,1),%r12 movq %rdx,%r13 adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[3]*a[0] + addq %rax,%r10 // a[3]*a[0]+a[2]*a[1]+t[3] movq %rbx,%rax adcq $0,%rdx addq %r12,%r10 movq %rdx,%r11 adcq $0,%r11 - movq %r10,-8(%rdi,%rbp,1) + movq %r10,-8(%rdi,%rbp,1) // t[3] leaq (%rbp),%rcx jmp L$sqr4x_inner .p2align 5 L$sqr4x_inner: - movq (%rsi,%rcx,1),%rbx - mulq %r15 - addq %rax,%r13 + movq (%rsi,%rcx,1),%rbx // a[4] + mulq %r15 // a[3]*a[1] + addq %rax,%r13 // a[3]*a[1]+t[4] movq %rbx,%rax movq %rdx,%r12 adcq $0,%r12 @@ -1493,254 +1493,254 @@ adcq $0,%r12 .byte 0x67 - mulq %r14 - addq %rax,%r11 - movq %rbx,%rax - movq 8(%rsi,%rcx,1),%rbx + mulq %r14 // a[4]*a[0] + addq %rax,%r11 // a[4]*a[0]+a[3]*a[1]+t[4] + movq %rbx,%rax // a[3] + movq 8(%rsi,%rcx,1),%rbx // a[5] movq %rdx,%r10 adcq $0,%r10 addq %r13,%r11 adcq $0,%r10 - mulq %r15 - addq %rax,%r12 - movq %r11,(%rdi,%rcx,1) + mulq %r15 // a[4]*a[3] + addq %rax,%r12 // a[4]*a[3]+t[5] + movq %r11,(%rdi,%rcx,1) // t[4] movq %rbx,%rax movq %rdx,%r13 adcq $0,%r13 addq 8(%rdi,%rcx,1),%r12 - leaq 16(%rcx),%rcx + leaq 16(%rcx),%rcx // j++ adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[5]*a[2] + addq %rax,%r10 // a[5]*a[2]+a[4]*a[3]+t[5] movq %rbx,%rax adcq $0,%rdx addq %r12,%r10 movq %rdx,%r11 adcq $0,%r11 - movq %r10,-8(%rdi,%rcx,1) + movq %r10,-8(%rdi,%rcx,1) // t[5], "preloaded t[1]" below cmpq $0,%rcx jne L$sqr4x_inner .byte 0x67 - mulq %r15 + mulq %r15 // a[5]*a[3] addq %rax,%r13 adcq $0,%rdx addq %r11,%r13 adcq $0,%rdx - movq %r13,(%rdi) + movq %r13,(%rdi) // t[6], "preloaded t[2]" below movq %rdx,%r12 - movq %rdx,8(%rdi) + movq %rdx,8(%rdi) // t[7], "preloaded t[3]" below addq $16,%rbp jnz L$sqr4x_outer - - movq -32(%rsi),%r14 - leaq 48+8(%rsp,%r9,2),%rdi - movq -24(%rsi),%rax - leaq -32(%rdi,%rbp,1),%rdi - movq -16(%rsi),%rbx +// comments apply to %r9==4 case + movq -32(%rsi),%r14 // a[0] + leaq 48+8(%rsp,%r9,2),%rdi // end of tp[] buffer, &tp[2*%r9] + movq -24(%rsi),%rax // a[1] + leaq -32(%rdi,%rbp,1),%rdi // end of tp[] window, &tp[2*%r9-"%rbp"] + movq -16(%rsi),%rbx // a[2] movq %rax,%r15 - mulq %r14 - addq %rax,%r10 - movq %rbx,%rax + mulq %r14 // a[1]*a[0] + addq %rax,%r10 // a[1]*a[0]+t[1], preloaded t[1] + movq %rbx,%rax // a[2] movq %rdx,%r11 adcq $0,%r11 - mulq %r14 + mulq %r14 // a[2]*a[0] addq %rax,%r11 movq %rbx,%rax - movq %r10,-24(%rdi) + movq %r10,-24(%rdi) // t[1] movq %rdx,%r10 adcq $0,%r10 - addq %r13,%r11 - movq -8(%rsi),%rbx + addq %r13,%r11 // a[2]*a[0]+t[2], preloaded t[2] + movq -8(%rsi),%rbx // a[3] adcq $0,%r10 - mulq %r15 - addq %rax,%r12 + mulq %r15 // a[2]*a[1] + addq %rax,%r12 // a[2]*a[1]+t[3], preloaded t[3] movq %rbx,%rax - movq %r11,-16(%rdi) + movq %r11,-16(%rdi) // t[2] movq %rdx,%r13 adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[3]*a[0] + addq %rax,%r10 // a[3]*a[0]+a[2]*a[1]+t[3] movq %rbx,%rax movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - movq %r10,-8(%rdi) + movq %r10,-8(%rdi) // t[3] - mulq %r15 + mulq %r15 // a[3]*a[1] addq %rax,%r13 - movq -16(%rsi),%rax + movq -16(%rsi),%rax // a[2] adcq $0,%rdx addq %r11,%r13 adcq $0,%rdx - movq %r13,(%rdi) + movq %r13,(%rdi) // t[4] movq %rdx,%r12 - movq %rdx,8(%rdi) + movq %rdx,8(%rdi) // t[5] - mulq %rbx + mulq %rbx // a[2]*a[3] addq $16,%rbp xorq %r14,%r14 - subq %r9,%rbp + subq %r9,%rbp // %rbp=16-%r9 xorq %r15,%r15 - addq %r12,%rax + addq %r12,%rax // t[5] adcq $0,%rdx - movq %rax,8(%rdi) - movq %rdx,16(%rdi) - movq %r15,24(%rdi) + movq %rax,8(%rdi) // t[5] + movq %rdx,16(%rdi) // t[6] + movq %r15,24(%rdi) // t[7] - movq -16(%rsi,%rbp,1),%rax + movq -16(%rsi,%rbp,1),%rax // a[0] leaq 48+8(%rsp),%rdi - xorq %r10,%r10 - movq 8(%rdi),%r11 + xorq %r10,%r10 // t[0] + movq 8(%rdi),%r11 // t[1] - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq 16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 24(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq 16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 24(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq -8(%rsi,%rbp,1),%rax + movq -8(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %r12,(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1 | shift movq %r13,8(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - movq 32(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 40(%rdi),%r11 + orq %r10,%r8 // | t[2*i]>>63 + movq 32(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 40(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%rbx - movq 0(%rsi,%rbp,1),%rax + movq 0(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %rbx,16(%rdi) adcq %rdx,%r8 leaq 16(%rbp),%rbp movq %r8,24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 leaq 64(%rdi),%rdi jmp L$sqr4x_shift_n_add .p2align 5 L$sqr4x_shift_n_add: - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq -16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq -8(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq -16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq -8(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq -8(%rsi,%rbp,1),%rax + movq -8(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %r12,-32(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1 | shift movq %r13,-24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - movq 0(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 8(%rdi),%r11 + orq %r10,%r8 // | t[2*i]>>63 + movq 0(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 8(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%rbx - movq 0(%rsi,%rbp,1),%rax + movq 0(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %rbx,-16(%rdi) adcq %rdx,%r8 - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift movq %r8,-8(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq 16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 24(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq 16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 24(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq 8(%rsi,%rbp,1),%rax + movq 8(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %r12,0(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1 | shift movq %r13,8(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - movq 32(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 40(%rdi),%r11 + orq %r10,%r8 // | t[2*i]>>63 + movq 32(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 40(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%rbx - movq 16(%rsi,%rbp,1),%rax + movq 16(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %rbx,16(%rdi) adcq %rdx,%r8 movq %r8,24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 leaq 64(%rdi),%rdi addq $32,%rbp jnz L$sqr4x_shift_n_add - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift .byte 0x67 shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq -16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq -8(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq -16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq -8(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq -8(%rsi),%rax + movq -8(%rsi),%rax // a[i+1] # prefetch movq %r12,-32(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1|shift movq %r13,-24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - mulq %rax - negq %r15 + orq %r10,%r8 // | t[2*i]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf adcq %rax,%rbx adcq %rdx,%r8 movq %rbx,-16(%rdi) @@ -1749,17 +1749,17 @@ .alt_entry __bn_sqr8x_reduction __bn_sqr8x_reduction: xorq %rax,%rax - leaq (%r9,%rbp,1),%rcx - leaq 48+8(%rsp,%r9,2),%rdx + leaq (%r9,%rbp,1),%rcx // end of n[] + leaq 48+8(%rsp,%r9,2),%rdx // end of t[] buffer movq %rcx,0+8(%rsp) - leaq 48+8(%rsp,%r9,1),%rdi + leaq 48+8(%rsp,%r9,1),%rdi // end of initial t[] window movq %rdx,8+8(%rsp) negq %r9 jmp L$8x_reduction_loop .p2align 5 L$8x_reduction_loop: - leaq (%rdi,%r9,1),%rdi + leaq (%rdi,%r9,1),%rdi // start of current t[] window .byte 0x66 movq 0(%rdi),%rbx movq 8(%rdi),%r9 @@ -1769,20 +1769,20 @@ movq 40(%rdi),%r13 movq 48(%rdi),%r14 movq 56(%rdi),%r15 - movq %rax,(%rdx) + movq %rax,(%rdx) // store top-most carry bit leaq 64(%rdi),%rdi .byte 0x67 movq %rbx,%r8 - imulq 32+8(%rsp),%rbx - movq 0(%rbp),%rax + imulq 32+8(%rsp),%rbx // n0*a[0] + movq 0(%rbp),%rax // n[0] movl $8,%ecx jmp L$8x_reduce .p2align 5 L$8x_reduce: mulq %rbx - movq 8(%rbp),%rax + movq 8(%rbp),%rax // n[1] negq %r8 movq %rdx,%r8 adcq $0,%r8 @@ -1792,7 +1792,7 @@ movq 16(%rbp),%rax adcq $0,%rdx addq %r9,%r8 - movq %rbx,48-8+8(%rsp,%rcx,8) + movq %rbx,48-8+8(%rsp,%rcx,8) // put aside n0*a[i] movq %rdx,%r9 adcq $0,%r9 @@ -1801,7 +1801,7 @@ movq 24(%rbp),%rax adcq $0,%rdx addq %r10,%r9 - movq 32+8(%rsp),%rsi + movq 32+8(%rsp),%rsi // pull n0, borrow %rsi movq %rdx,%r10 adcq $0,%r10 @@ -1809,7 +1809,7 @@ addq %rax,%r11 movq 32(%rbp),%rax adcq $0,%rdx - imulq %r8,%rsi + imulq %r8,%rsi // modulo-scheduled addq %r11,%r10 movq %rdx,%r11 adcq $0,%r11 @@ -1839,9 +1839,9 @@ adcq $0,%r14 mulq %rbx - movq %rsi,%rbx + movq %rsi,%rbx // n0*a[i] addq %rax,%r15 - movq 0(%rbp),%rax + movq 0(%rbp),%rax // n[0] adcq $0,%rdx addq %r15,%r14 movq %rdx,%r15 @@ -1852,8 +1852,8 @@ leaq 64(%rbp),%rbp xorq %rax,%rax - movq 8+8(%rsp),%rdx - cmpq 0+8(%rsp),%rbp + movq 8+8(%rsp),%rdx // pull end of t[] + cmpq 0+8(%rsp),%rbp // end of n[]? jae L$8x_no_tail .byte 0x66 @@ -1865,9 +1865,9 @@ adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - sbbq %rsi,%rsi + sbbq %rsi,%rsi // top carry - movq 48+56+8(%rsp),%rbx + movq 48+56+8(%rsp),%rbx // pull n0*a[0] movl $8,%ecx movq 0(%rbp),%rax jmp L$8x_tail @@ -1877,7 +1877,7 @@ mulq %rbx addq %rax,%r8 movq 8(%rbp),%rax - movq %r8,(%rdi) + movq %r8,(%rdi) // save result movq %rdx,%r8 adcq $0,%r8 @@ -1886,7 +1886,7 @@ movq 16(%rbp),%rax adcq $0,%rdx addq %r9,%r8 - leaq 8(%rdi),%rdi + leaq 8(%rdi),%rdi // %rdi++ movq %rdx,%r9 adcq $0,%r9 @@ -1931,11 +1931,11 @@ adcq $0,%r14 mulq %rbx - movq 48-16+8(%rsp,%rcx,8),%rbx + movq 48-16+8(%rsp,%rcx,8),%rbx // pull n0*a[i] addq %rax,%r15 adcq $0,%rdx addq %r15,%r14 - movq 0(%rbp),%rax + movq 0(%rbp),%rax // pull n[0] movq %rdx,%r15 adcq $0,%r15 @@ -1943,13 +1943,13 @@ jnz L$8x_tail leaq 64(%rbp),%rbp - movq 8+8(%rsp),%rdx - cmpq 0+8(%rsp),%rbp - jae L$8x_tail_done + movq 8+8(%rsp),%rdx // pull end of t[] + cmpq 0+8(%rsp),%rbp // end of n[]? + jae L$8x_tail_done // break out of loop - movq 48+56+8(%rsp),%rbx + movq 48+56+8(%rsp),%rbx // pull n0*a[0] negq %rsi - movq 0(%rbp),%rax + movq 0(%rbp),%rax // pull n[0] adcq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 @@ -1958,7 +1958,7 @@ adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - sbbq %rsi,%rsi + sbbq %rsi,%rsi // top carry movl $8,%ecx jmp L$8x_tail @@ -1966,7 +1966,7 @@ .p2align 5 L$8x_tail_done: xorq %rax,%rax - addq (%rdx),%r8 + addq (%rdx),%r8 // can this overflow? adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 @@ -1986,15 +1986,15 @@ adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - adcq $0,%rax - movq -8(%rbp),%rcx + adcq $0,%rax // top-most carry + movq -8(%rbp),%rcx // np[num-1] xorq %rsi,%rsi - movq %xmm2,%rbp + movq %xmm2,%rbp // restore %rbp - movq %r8,0(%rdi) + movq %r8,0(%rdi) // store top 512 bits movq %r9,8(%rdi) - movq %xmm3,%r9 + movq %xmm3,%r9 // %r9 is %r9, can't be moved upwards movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) @@ -2003,7 +2003,7 @@ movq %r15,56(%rdi) leaq 64(%rdi),%rdi - cmpq %rdx,%rdi + cmpq %rdx,%rdi // end of t[]? jb L$8x_reduction_loop ret @@ -2014,13 +2014,13 @@ __bn_post4x_internal: movq 0(%rbp),%r12 - leaq (%rdi,%r9,1),%rbx + leaq (%rdi,%r9,1),%rbx // %rdi was %rbx above movq %r9,%rcx - movq %xmm1,%rdi + movq %xmm1,%rdi // restore %rdi negq %rax - movq %xmm1,%rsi + movq %xmm1,%rsi // prepare for back-to-back call sarq $3+2,%rcx - decq %r12 + decq %r12 // so that after 'not' we get -n[0] xorq %r10,%r10 movq 8(%rbp),%r13 movq 16(%rbp),%r14 @@ -2044,7 +2044,7 @@ andq %rax,%r14 andq %rax,%r15 - negq %r10 + negq %r10 // mov %r10,%cf adcq 0(%rbx),%r12 adcq 8(%rbx),%r13 adcq 16(%rbx),%r14 @@ -2052,16 +2052,16 @@ movq %r12,0(%rdi) leaq 32(%rbx),%rbx movq %r13,8(%rdi) - sbbq %r10,%r10 + sbbq %r10,%r10 // mov %cf,%r10 movq %r14,16(%rdi) movq %r15,24(%rdi) leaq 32(%rdi),%rdi - incq %rcx + incq %rcx // pass %cf jnz L$sqr4x_sub - movq %r9,%r10 - negq %r9 + movq %r9,%r10 // prepare for back-to-back call + negq %r9 // restore %r9 ret @@ -2089,42 +2089,42 @@ L$mulx4x_prologue: +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, predates the first access. + shll $3,%r9d // convert %r9 to bytes + leaq (%r9,%r9,2),%r10 // 3*%r9 in bytes + negq %r9 // -%r9 + movq (%r8),%r8 // *n0 - - - shll $3,%r9d - leaq (%r9,%r9,2),%r10 - negq %r9 - movq (%r8),%r8 - - - - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with +3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra [num] is allocated in order +// to align with bn_power5's frame, which is cleansed after +// completing exponentiation. Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb L$mulx4xsp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9*8+256) jmp L$mulx4xsp_done L$mulx4xsp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 subq %r11,%rbp L$mulx4xsp_done: - andq $-64,%rbp + andq $-64,%rbp // ensure alignment movq %rsp,%r11 subq %rbp,%r11 andq $-4096,%r11 @@ -2141,25 +2141,25 @@ ja L$mulx4x_page_walk L$mulx4x_page_walk_done: - - - - - - - - - - - - - movq %r8,32(%rsp) - movq %rax,40(%rsp) +// ############################################################# +// Stack layout +// +0 -num +// +8 off-loaded &b[i] +// +16 end of b[num] +// +24 inner counter +// +32 saved n0 +// +40 saved %rsp +// +48 +// +56 saved rp +// +64 tmp[num+1] +// + movq %r8,32(%rsp) // save *n0 + movq %rax,40(%rsp) // save original %rsp L$mulx4x_body: call mulx4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp movq $1,%rax @@ -2187,44 +2187,44 @@ .alt_entry mulx4x_internal mulx4x_internal: - movq %r9,8(%rsp) + movq %r9,8(%rsp) // save -%r9 (it was in bytes) movq %r9,%r10 - negq %r9 + negq %r9 // restore %r9 shlq $5,%r9 - negq %r10 - leaq 128(%rdx,%r9,1),%r13 + negq %r10 // restore %r9 + leaq 128(%rdx,%r9,1),%r13 // end of powers table (+size optimization) shrq $5+5,%r9 - movd 8(%rax),%xmm5 + movd 8(%rax),%xmm5 // load 7th argument subq $1,%r9 leaq L$inc(%rip),%rax - movq %r13,16+8(%rsp) - movq %r9,24+8(%rsp) - movq %rdi,56+8(%rsp) - movdqa 0(%rax),%xmm0 - movdqa 16(%rax),%xmm1 - leaq 88-112(%rsp,%r10,1),%r10 - leaq 128(%rdx),%rdi + movq %r13,16+8(%rsp) // end of b[num] + movq %r9,24+8(%rsp) // inner counter + movq %rdi,56+8(%rsp) // save %rdi + movdqa 0(%rax),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%rax),%xmm1 // 00000002000000020000000200000002 + leaq 88-112(%rsp,%r10,1),%r10 // place the mask after tp[num+1] (+ICache optimization) + leaq 128(%rdx),%rdi // size optimization - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast index movdqa %xmm1,%xmm4 .byte 0x67 movdqa %xmm1,%xmm2 .byte 0x67 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,112(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,128(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,144(%r10) movdqa %xmm4,%xmm2 @@ -2233,17 +2233,17 @@ movdqa %xmm3,160(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,176(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,192(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,208(%r10) movdqa %xmm4,%xmm2 @@ -2252,17 +2252,17 @@ movdqa %xmm3,224(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,240(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,256(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,272(%r10) movdqa %xmm4,%xmm2 @@ -2282,7 +2282,7 @@ pcmpeqd %xmm5,%xmm3 movdqa %xmm2,336(%r10) - pand 64(%rdi),%xmm0 + pand 64(%rdi),%xmm0 // while it's still in register pand 80(%rdi),%xmm1 pand 96(%rdi),%xmm2 movdqa %xmm3,352(%r10) @@ -2326,41 +2326,41 @@ por %xmm2,%xmm0 por %xmm3,%xmm1 pxor %xmm1,%xmm0 - - pshufd $0x4e,%xmm0,%xmm1 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm0,%xmm1 // Swap upper and lower halves. por %xmm1,%xmm0 leaq 256(%rdi),%rdi - movq %xmm0,%rdx + movq %xmm0,%rdx // bp[0] leaq 64+32+8(%rsp),%rbx movq %rdx,%r9 - mulxq 0(%rsi),%r8,%rax - mulxq 8(%rsi),%r11,%r12 + mulxq 0(%rsi),%r8,%rax // a[0]*b[0] + mulxq 8(%rsi),%r11,%r12 // a[1]*b[0] addq %rax,%r11 - mulxq 16(%rsi),%rax,%r13 + mulxq 16(%rsi),%rax,%r13 // ... adcq %rax,%r12 adcq $0,%r13 mulxq 24(%rsi),%rax,%r14 movq %r8,%r15 - imulq 32+8(%rsp),%r8 - xorq %rbp,%rbp + imulq 32+8(%rsp),%r8 // "t[0]"*n0 + xorq %rbp,%rbp // cf=0, of=0 movq %r8,%rdx - movq %rdi,8+8(%rsp) + movq %rdi,8+8(%rsp) // off-load &b[i] leaq 32(%rsi),%rsi adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%r15 + adcxq %rax,%r15 // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 adoxq %r12,%r11 mulxq 16(%rcx),%rax,%r12 - movq 24+8(%rsp),%rdi + movq 24+8(%rsp),%rdi // counter value movq %r10,-32(%rbx) adcxq %rax,%r11 adoxq %r13,%r12 @@ -2368,25 +2368,25 @@ movq %r9,%rdx movq %r11,-24(%rbx) adcxq %rax,%r12 - adoxq %rbp,%r15 + adoxq %rbp,%r15 // of=0 leaq 32(%rcx),%rcx movq %r12,-16(%rbx) jmp L$mulx4x_1st .p2align 5 L$mulx4x_1st: - adcxq %rbp,%r15 - mulxq 0(%rsi),%r10,%rax + adcxq %rbp,%r15 // cf=0, modulo-scheduled + mulxq 0(%rsi),%r10,%rax // a[4]*b[0] adcxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[0] adcxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 .byte 0x67,0x67 movq %r8,%rdx adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx @@ -2410,21 +2410,21 @@ leaq 32(%rcx),%rcx movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz L$mulx4x_1st - movq 8(%rsp),%rax - adcq %rbp,%r15 - leaq (%rsi,%rax,1),%rsi + movq 8(%rsp),%rax // load -num + adcq %rbp,%r15 // modulo-scheduled + leaq (%rsi,%rax,1),%rsi // rewind %rsi addq %r15,%r14 - movq 8+8(%rsp),%rdi - adcq %rbp,%rbp + movq 8+8(%rsp),%rdi // re-load &b[i] + adcq %rbp,%rbp // top-most carry movq %r14,-8(%rbx) jmp L$mulx4x_outer .p2align 5 L$mulx4x_outer: - leaq 16-256(%rbx),%r10 + leaq 16-256(%rbx),%r10 // where 256-byte mask is (+density control) pxor %xmm4,%xmm4 .byte 0x67,0x67 pxor %xmm5,%xmm5 @@ -2477,41 +2477,41 @@ por %xmm2,%xmm4 por %xmm3,%xmm5 por %xmm5,%xmm4 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 leaq 256(%rdi),%rdi - movq %xmm0,%rdx + movq %xmm0,%rdx // m0=bp[i] - movq %rbp,(%rbx) - leaq 32(%rbx,%rax,1),%rbx - mulxq 0(%rsi),%r8,%r11 - xorq %rbp,%rbp + movq %rbp,(%rbx) // save top-most carry + leaq 32(%rbx,%rax,1),%rbx // rewind %rbx + mulxq 0(%rsi),%r8,%r11 // a[0]*b[i] + xorq %rbp,%rbp // cf=0, of=0 movq %rdx,%r9 - mulxq 8(%rsi),%r14,%r12 - adoxq -32(%rbx),%r8 + mulxq 8(%rsi),%r14,%r12 // a[1]*b[i] + adoxq -32(%rbx),%r8 // +t[0] adcxq %r14,%r11 - mulxq 16(%rsi),%r15,%r13 + mulxq 16(%rsi),%r15,%r13 // ... adoxq -24(%rbx),%r11 adcxq %r15,%r12 mulxq 24(%rsi),%rdx,%r14 adoxq -16(%rbx),%r12 adcxq %rdx,%r13 - leaq (%rcx,%rax,1),%rcx + leaq (%rcx,%rax,1),%rcx // rewind %rcx leaq 32(%rsi),%rsi adoxq -8(%rbx),%r13 adcxq %rbp,%r14 adoxq %rbp,%r14 movq %r8,%r15 - imulq 32+8(%rsp),%r8 + imulq 32+8(%rsp),%r8 // "t[0]"*n0 movq %r8,%rdx - xorq %rbp,%rbp - movq %rdi,8+8(%rsp) + xorq %rbp,%rbp // cf=0, of=0 + movq %rdi,8+8(%rsp) // off-load &b[i] mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%r15 + adcxq %rax,%r15 // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 @@ -2521,24 +2521,24 @@ adoxq %r13,%r12 mulxq 24(%rcx),%rax,%r15 movq %r9,%rdx - movq 24+8(%rsp),%rdi + movq 24+8(%rsp),%rdi // counter value movq %r10,-32(%rbx) adcxq %rax,%r12 movq %r11,-24(%rbx) - adoxq %rbp,%r15 + adoxq %rbp,%r15 // of=0 movq %r12,-16(%rbx) leaq 32(%rcx),%rcx jmp L$mulx4x_inner .p2align 5 L$mulx4x_inner: - mulxq 0(%rsi),%r10,%rax - adcxq %rbp,%r15 + mulxq 0(%rsi),%r10,%rax // a[4]*b[i] + adcxq %rbp,%r15 // cf=0, modulo-scheduled adoxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[i] adcxq 0(%rbx),%r10 adoxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq 8(%rbx),%r11 adoxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 @@ -2546,10 +2546,10 @@ adcxq 16(%rbx),%r12 adoxq %rax,%r13 adcxq 24(%rbx),%r13 - adoxq %rbp,%r14 + adoxq %rbp,%r14 // of=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 adoxq %r15,%r10 mulxq 0(%rcx),%rax,%r15 @@ -2571,17 +2571,17 @@ adoxq %rbp,%r15 movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz L$mulx4x_inner - movq 0+8(%rsp),%rax - adcq %rbp,%r15 - subq 0(%rbx),%rdi - movq 8+8(%rsp),%rdi + movq 0+8(%rsp),%rax // load -num + adcq %rbp,%r15 // modulo-scheduled + subq 0(%rbx),%rdi // pull top-most carry to %cf + movq 8+8(%rsp),%rdi // re-load &b[i] movq 16+8(%rsp),%r10 adcq %r15,%r14 - leaq (%rsi,%rax,1),%rsi - adcq %rbp,%rbp + leaq (%rsi,%rax,1),%rsi // rewind %rsi + adcq %rbp,%rbp // top-most carry movq %r14,-8(%rbx) cmpq %r10,%rdi @@ -2590,23 +2590,23 @@ movq -8(%rcx),%r10 movq %rbp,%r8 movq (%rcx,%rax,1),%r12 - leaq (%rcx,%rax,1),%rbp + leaq (%rcx,%rax,1),%rbp // rewind %rcx movq %rax,%rcx - leaq (%rbx,%rax,1),%rdi + leaq (%rbx,%rax,1),%rdi // rewind %rbx xorl %eax,%eax xorq %r15,%r15 - subq %r14,%r10 + subq %r14,%r10 // compare top-most words adcq %r15,%r15 orq %r15,%r8 sarq $3+2,%rcx - subq %r8,%rax - movq 56+8(%rsp),%rdx - decq %r12 + subq %r8,%rax // %rax=-%r8 + movq 56+8(%rsp),%rdx // restore rp + decq %r12 // so that after 'not' we get -n[0] movq 8(%rbp),%r13 xorq %r8,%r8 movq 16(%rbp),%r14 movq 24(%rbp),%r15 - jmp L$sqrx4x_sub_entry + jmp L$sqrx4x_sub_entry // common post-condition .globl _bn_powerx5 @@ -2633,35 +2633,35 @@ L$powerx5_prologue: - - - - shll $3,%r9d - leaq (%r9,%r9,2),%r10 +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, predates the first access. + shll $3,%r9d // convert %r9 to bytes + leaq (%r9,%r9,2),%r10 // 3*%r9 in bytes negq %r9 - movq (%r8),%r8 + movq (%r8),%r8 // *n0 - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with %rdi+3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb L$pwrx_sp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rsi + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9*8+256) jmp L$pwrx_sp_done .p2align 5 L$pwrx_sp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // alloca(frame+2*%r9*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -2687,24 +2687,24 @@ movq %r9,%r10 negq %r9 - - - - - - - - - - - +// ############################################################# +// Stack layout +// +// +0 saved %r9, used in reduction section +// +8 &t[2*%r9], used in reduction section +// +16 intermediate carry bit +// +24 top-most carry bit, used in reduction section +// +32 saved *n0 +// +40 saved %rsp +// +48 t[2*%r9] +// pxor %xmm0,%xmm0 - movq %rdi,%xmm1 - movq %rcx,%xmm2 - movq %r10,%xmm3 + movq %rdi,%xmm1 // save %rdi + movq %rcx,%xmm2 // save %rcx + movq %r10,%xmm3 // -%r9 movq %rdx,%xmm4 movq %r8,32(%rsp) - movq %rax,40(%rsp) + movq %rax,40(%rsp) // save original %rsp L$powerx5_body: @@ -2719,7 +2719,7 @@ call __bn_sqrx8x_internal call __bn_postx4x_internal - movq %r10,%r9 + movq %r10,%r9 // -num movq %rsi,%rdi movq %xmm2,%rcx movq %xmm4,%rdx @@ -2727,7 +2727,7 @@ call mulx4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp movq $1,%rax @@ -2761,50 +2761,50 @@ __bn_sqrx8x_internal: _CET_ENDBR - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// ################################################################# +// Squaring part: +// +// a) multiply-n-add everything but a[i]*a[i]; +// b) shift result of a) by 1 to the left and accumulate +// a[i]*a[i] products; +// +// ################################################################# +// a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] +// a[1]a[0] +// a[2]a[0] +// a[3]a[0] +// a[2]a[1] +// a[3]a[1] +// a[3]a[2] +// +// a[4]a[0] +// a[5]a[0] +// a[6]a[0] +// a[7]a[0] +// a[4]a[1] +// a[5]a[1] +// a[6]a[1] +// a[7]a[1] +// a[4]a[2] +// a[5]a[2] +// a[6]a[2] +// a[7]a[2] +// a[4]a[3] +// a[5]a[3] +// a[6]a[3] +// a[7]a[3] +// +// a[5]a[4] +// a[6]a[4] +// a[7]a[4] +// a[6]a[5] +// a[7]a[5] +// a[7]a[6] +// a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] leaq 48+8(%rsp),%rdi leaq (%rsi,%r9,1),%rbp - movq %r9,0+8(%rsp) - movq %rbp,8+8(%rsp) + movq %r9,0+8(%rsp) // save %r9 + movq %rbp,8+8(%rsp) // save end of %rsi jmp L$sqr8x_zero_start .p2align 5 @@ -2815,7 +2815,7 @@ movdqa %xmm0,16(%rdi) movdqa %xmm0,32(%rdi) movdqa %xmm0,48(%rdi) -L$sqr8x_zero_start: +L$sqr8x_zero_start: // aligned at 32 movdqa %xmm0,64(%rdi) movdqa %xmm0,80(%rdi) movdqa %xmm0,96(%rdi) @@ -2824,8 +2824,8 @@ subq $64,%r9 jnz L$sqrx8x_zero - movq 0(%rsi),%rdx - + movq 0(%rsi),%rdx // a[0], modulo-scheduled +// xor %r9,%r9 # t[1], ex-%r9, zero already xorq %r10,%r10 xorq %r11,%r11 xorq %r12,%r12 @@ -2833,21 +2833,21 @@ xorq %r14,%r14 xorq %r15,%r15 leaq 48+8(%rsp),%rdi - xorq %rbp,%rbp + xorq %rbp,%rbp // cf=0, cf=0 jmp L$sqrx8x_outer_loop .p2align 5 L$sqrx8x_outer_loop: - mulxq 8(%rsi),%r8,%rax - adcxq %r9,%r8 + mulxq 8(%rsi),%r8,%rax // a[1]*a[0] + adcxq %r9,%r8 // a[1]*a[0]+=t[1] adoxq %rax,%r10 - mulxq 16(%rsi),%r9,%rax + mulxq 16(%rsi),%r9,%rax // a[2]*a[0] adcxq %r10,%r9 adoxq %rax,%r11 -.byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 // mulx 3*8(%rsi),%r10,%rax # ... adcxq %r11,%r10 adoxq %rax,%r12 -.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 // mulx 4*8(%rsi),%r11,%rax adcxq %r12,%r11 adoxq %rax,%r13 mulxq 40(%rsi),%r12,%rax @@ -2857,97 +2857,97 @@ adcxq %r14,%r13 adoxq %r15,%rax mulxq 56(%rsi),%r14,%r15 - movq 8(%rsi),%rdx + movq 8(%rsi),%rdx // a[1] adcxq %rax,%r14 adoxq %rbp,%r15 adcq 64(%rdi),%r15 - movq %r8,8(%rdi) - movq %r9,16(%rdi) - sbbq %rcx,%rcx - xorq %rbp,%rbp + movq %r8,8(%rdi) // t[1] + movq %r9,16(%rdi) // t[2] + sbbq %rcx,%rcx // mov %cf,%rcx + xorq %rbp,%rbp // cf=0, of=0 - mulxq 16(%rsi),%r8,%rbx - mulxq 24(%rsi),%r9,%rax + mulxq 16(%rsi),%r8,%rbx // a[2]*a[1] + mulxq 24(%rsi),%r9,%rax // a[3]*a[1] adcxq %r10,%r8 adoxq %rbx,%r9 - mulxq 32(%rsi),%r10,%rbx + mulxq 32(%rsi),%r10,%rbx // ... adcxq %r11,%r9 adoxq %rax,%r10 -.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 // mulx 5*8(%rsi),%r11,%rax adcxq %r12,%r10 adoxq %rbx,%r11 -.byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 +.byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 // mulx 6*8(%rsi),%r12,%rbx adcxq %r13,%r11 adoxq %r14,%r12 -.byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 - movq 16(%rsi),%rdx +.byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 // mulx 7*8(%rsi),%r13,%r14 + movq 16(%rsi),%rdx // a[2] adcxq %rax,%r12 adoxq %rbx,%r13 adcxq %r15,%r13 - adoxq %rbp,%r14 - adcxq %rbp,%r14 + adoxq %rbp,%r14 // of=0 + adcxq %rbp,%r14 // cf=0 - movq %r8,24(%rdi) - movq %r9,32(%rdi) + movq %r8,24(%rdi) // t[3] + movq %r9,32(%rdi) // t[4] - mulxq 24(%rsi),%r8,%rbx - mulxq 32(%rsi),%r9,%rax + mulxq 24(%rsi),%r8,%rbx // a[3]*a[2] + mulxq 32(%rsi),%r9,%rax // a[4]*a[2] adcxq %r10,%r8 adoxq %rbx,%r9 - mulxq 40(%rsi),%r10,%rbx + mulxq 40(%rsi),%r10,%rbx // ... adcxq %r11,%r9 adoxq %rax,%r10 -.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 // mulx 6*8(%rsi),%r11,%rax adcxq %r12,%r10 adoxq %r13,%r11 -.byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 +.byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 // mulx 7*8(%rsi),%r12,%r13 .byte 0x3e - movq 24(%rsi),%rdx + movq 24(%rsi),%rdx // a[3] adcxq %rbx,%r11 adoxq %rax,%r12 adcxq %r14,%r12 - movq %r8,40(%rdi) - movq %r9,48(%rdi) - mulxq 32(%rsi),%r8,%rax - adoxq %rbp,%r13 - adcxq %rbp,%r13 + movq %r8,40(%rdi) // t[5] + movq %r9,48(%rdi) // t[6] + mulxq 32(%rsi),%r8,%rax // a[4]*a[3] + adoxq %rbp,%r13 // of=0 + adcxq %rbp,%r13 // cf=0 - mulxq 40(%rsi),%r9,%rbx + mulxq 40(%rsi),%r9,%rbx // a[5]*a[3] adcxq %r10,%r8 adoxq %rax,%r9 - mulxq 48(%rsi),%r10,%rax + mulxq 48(%rsi),%r10,%rax // ... adcxq %r11,%r9 adoxq %r12,%r10 mulxq 56(%rsi),%r11,%r12 - movq 32(%rsi),%rdx - movq 40(%rsi),%r14 + movq 32(%rsi),%rdx // a[4] + movq 40(%rsi),%r14 // a[5] adcxq %rbx,%r10 adoxq %rax,%r11 - movq 48(%rsi),%r15 + movq 48(%rsi),%r15 // a[6] adcxq %r13,%r11 - adoxq %rbp,%r12 - adcxq %rbp,%r12 + adoxq %rbp,%r12 // of=0 + adcxq %rbp,%r12 // cf=0 - movq %r8,56(%rdi) - movq %r9,64(%rdi) + movq %r8,56(%rdi) // t[7] + movq %r9,64(%rdi) // t[8] - mulxq %r14,%r9,%rax - movq 56(%rsi),%r8 + mulxq %r14,%r9,%rax // a[5]*a[4] + movq 56(%rsi),%r8 // a[7] adcxq %r10,%r9 - mulxq %r15,%r10,%rbx + mulxq %r15,%r10,%rbx // a[6]*a[4] adoxq %rax,%r10 adcxq %r11,%r10 - mulxq %r8,%r11,%rax - movq %r14,%rdx + mulxq %r8,%r11,%rax // a[7]*a[4] + movq %r14,%rdx // a[5] adoxq %rbx,%r11 adcxq %r12,%r11 +// adox %rbp,%rax # of=0 + adcxq %rbp,%rax // cf=0 - adcxq %rbp,%rax - - mulxq %r15,%r14,%rbx - mulxq %r8,%r12,%r13 - movq %r15,%rdx + mulxq %r15,%r14,%rbx // a[6]*a[5] + mulxq %r8,%r12,%r13 // a[7]*a[5] + movq %r15,%rdx // a[6] leaq 64(%rsi),%rsi adcxq %r14,%r11 adoxq %rbx,%r12 @@ -2955,19 +2955,19 @@ adoxq %rbp,%r13 .byte 0x67,0x67 - mulxq %r8,%r8,%r14 + mulxq %r8,%r8,%r14 // a[7]*a[6] adcxq %r8,%r13 adcxq %rbp,%r14 cmpq 8+8(%rsp),%rsi je L$sqrx8x_outer_break - negq %rcx + negq %rcx // mov %rcx,%cf movq $-8,%rcx movq %rbp,%r15 movq 64(%rdi),%r8 - adcxq 72(%rdi),%r9 - adcxq 80(%rdi),%r10 + adcxq 72(%rdi),%r9 // +=t[9] + adcxq 80(%rdi),%r10 // ... adcxq 88(%rdi),%r11 adcq 96(%rdi),%r12 adcq 104(%rdi),%r13 @@ -2975,24 +2975,24 @@ adcq 120(%rdi),%r15 leaq (%rsi),%rbp leaq 128(%rdi),%rdi - sbbq %rax,%rax + sbbq %rax,%rax // mov %cf,%rcx - movq -64(%rsi),%rdx - movq %rax,16+8(%rsp) + movq -64(%rsi),%rdx // a[0] + movq %rax,16+8(%rsp) // offload %rcx movq %rdi,24+8(%rsp) - - xorl %eax,%eax +// lea 8*8(%rdi),%rdi # see 2*8*8(%rdi) above + xorl %eax,%eax // cf=0, of=0 jmp L$sqrx8x_loop .p2align 5 L$sqrx8x_loop: movq %r8,%rbx - mulxq 0(%rbp),%rax,%r8 - adcxq %rax,%rbx + mulxq 0(%rbp),%rax,%r8 // a[8]*a[i] + adcxq %rax,%rbx // +=t[8] adoxq %r9,%r8 - mulxq 8(%rbp),%rax,%r9 + mulxq 8(%rbp),%rax,%r9 // ... adcxq %rax,%r8 adoxq %r10,%r9 @@ -3004,7 +3004,7 @@ adcxq %rax,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 +.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 // mulx 4*8(%rbp),%rax,%r12 adcxq %rax,%r11 adoxq %r13,%r12 @@ -3013,27 +3013,27 @@ adoxq %r14,%r13 mulxq 48(%rbp),%rax,%r14 - movq %rbx,(%rdi,%rcx,8) + movq %rbx,(%rdi,%rcx,8) // store t[8+i] movl $0,%ebx adcxq %rax,%r13 adoxq %r15,%r14 -.byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 - movq 8(%rsi,%rcx,8),%rdx +.byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 // mulx 7*8(%rbp),%rax,%r15 + movq 8(%rsi,%rcx,8),%rdx // a[i] adcxq %rax,%r14 - adoxq %rbx,%r15 - adcxq %rbx,%r15 + adoxq %rbx,%r15 // %rbx is 0, of=0 + adcxq %rbx,%r15 // cf=0 .byte 0x67 - incq %rcx + incq %rcx // of=0 jnz L$sqrx8x_loop leaq 64(%rbp),%rbp movq $-8,%rcx - cmpq 8+8(%rsp),%rbp + cmpq 8+8(%rsp),%rbp // done? je L$sqrx8x_break - subq 16+8(%rsp),%rbx + subq 16+8(%rsp),%rbx // mov 16(%rsp),%cf .byte 0x66 movq -64(%rsi),%rdx adcxq 0(%rdi),%r8 @@ -3046,19 +3046,19 @@ adcq 56(%rdi),%r15 leaq 64(%rdi),%rdi .byte 0x67 - sbbq %rax,%rax - xorl %ebx,%ebx - movq %rax,16+8(%rsp) + sbbq %rax,%rax // mov %cf,%rax + xorl %ebx,%ebx // cf=0, of=0 + movq %rax,16+8(%rsp) // offload carry jmp L$sqrx8x_loop .p2align 5 L$sqrx8x_break: xorq %rbp,%rbp - subq 16+8(%rsp),%rbx + subq 16+8(%rsp),%rbx // mov 16(%rsp),%cf adcxq %rbp,%r8 - movq 24+8(%rsp),%rcx + movq 24+8(%rsp),%rcx // initial %rdi, borrow %rcx adcxq %rbp,%r9 - movq 0(%rsi),%rdx + movq 0(%rsi),%rdx // a[8], modulo-scheduled adcq $0,%r10 movq %r8,0(%rdi) adcq $0,%r11 @@ -3066,7 +3066,7 @@ adcq $0,%r13 adcq $0,%r14 adcq $0,%r15 - cmpq %rcx,%rdi + cmpq %rcx,%rdi // cf=0, of=0 je L$sqrx8x_outer_loop movq %r9,8(%rdi) @@ -3088,57 +3088,57 @@ .p2align 5 L$sqrx8x_outer_break: - movq %r9,72(%rdi) - movq %xmm3,%rcx - movq %r10,80(%rdi) + movq %r9,72(%rdi) // t[9] + movq %xmm3,%rcx // -%r9 + movq %r10,80(%rdi) // ... movq %r11,88(%rdi) movq %r12,96(%rdi) movq %r13,104(%rdi) movq %r14,112(%rdi) leaq 48+8(%rsp),%rdi - movq (%rsi,%rcx,1),%rdx + movq (%rsi,%rcx,1),%rdx // a[0] - movq 8(%rdi),%r11 - xorq %r10,%r10 - movq 0+8(%rsp),%r9 + movq 8(%rdi),%r11 // t[1] + xorq %r10,%r10 // t[0], of=0, cf=0 + movq 0+8(%rsp),%r9 // restore %r9 adoxq %r11,%r11 - movq 16(%rdi),%r12 - movq 24(%rdi),%r13 - + movq 16(%rdi),%r12 // t[2] # prefetch + movq 24(%rdi),%r13 // t[3] # prefetch +// jmp .Lsqrx4x_shift_n_add # happens to be aligned .p2align 5 L$sqrx4x_shift_n_add: mulxq %rdx,%rax,%rbx adoxq %r12,%r12 adcxq %r10,%rax -.byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 -.byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 +.byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 // mov 8(%rsi,%rcx),%rdx # a[i+1] # prefetch +.byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 // mov 32(%rdi),%r10 # t[2*i+4] # prefetch adoxq %r13,%r13 adcxq %r11,%rbx - movq 40(%rdi),%r11 + movq 40(%rdi),%r11 // t[2*i+4+1] # prefetch movq %rax,0(%rdi) movq %rbx,8(%rdi) mulxq %rdx,%rax,%rbx adoxq %r10,%r10 adcxq %r12,%rax - movq 16(%rsi,%rcx,1),%rdx - movq 48(%rdi),%r12 + movq 16(%rsi,%rcx,1),%rdx // a[i+2] # prefetch + movq 48(%rdi),%r12 // t[2*i+6] # prefetch adoxq %r11,%r11 adcxq %r13,%rbx - movq 56(%rdi),%r13 + movq 56(%rdi),%r13 // t[2*i+6+1] # prefetch movq %rax,16(%rdi) movq %rbx,24(%rdi) mulxq %rdx,%rax,%rbx adoxq %r12,%r12 adcxq %r10,%rax - movq 24(%rsi,%rcx,1),%rdx + movq 24(%rsi,%rcx,1),%rdx // a[i+3] # prefetch leaq 32(%rcx),%rcx - movq 64(%rdi),%r10 + movq 64(%rdi),%r10 // t[2*i+8] # prefetch adoxq %r13,%r13 adcxq %r11,%rbx - movq 72(%rdi),%r11 + movq 72(%rdi),%r11 // t[2*i+8+1] # prefetch movq %rax,32(%rdi) movq %rbx,40(%rdi) @@ -3146,11 +3146,11 @@ adoxq %r10,%r10 adcxq %r12,%rax jrcxz L$sqrx4x_shift_n_add_break -.byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 +.byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 // mov 0(%rsi,%rcx),%rdx # a[i+4] # prefetch adoxq %r11,%r11 adcxq %r13,%rbx - movq 80(%rdi),%r12 - movq 88(%rdi),%r13 + movq 80(%rdi),%r12 // t[2*i+10] # prefetch + movq 88(%rdi),%r13 // t[2*i+10+1] # prefetch movq %rax,48(%rdi) movq %rbx,56(%rdi) leaq 64(%rdi),%rdi @@ -3162,19 +3162,19 @@ adcxq %r13,%rbx movq %rax,48(%rdi) movq %rbx,56(%rdi) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // end of t[] buffer movq %xmm2,%rbp .alt_entry __bn_sqrx8x_reduction __bn_sqrx8x_reduction: - xorl %eax,%eax - movq 32+8(%rsp),%rbx - movq 48+8(%rsp),%rdx - leaq -64(%rbp,%r9,1),%rcx + xorl %eax,%eax // initial top-most carry bit + movq 32+8(%rsp),%rbx // n0 + movq 48+8(%rsp),%rdx // "%r8", 8*0(%rdi) + leaq -64(%rbp,%r9,1),%rcx // end of n[] +// lea 48+8(%rsp,%r9,2),%rdi # end of t[] buffer + movq %rcx,0+8(%rsp) // save end of n[] + movq %rdi,8+8(%rsp) // save end of t[] - movq %rcx,0+8(%rsp) - movq %rdi,8+8(%rsp) - - leaq 48+8(%rsp),%rdi + leaq 48+8(%rsp),%rdi // initial t[] window jmp L$sqrx8x_reduction_loop .p2align 5 @@ -3184,25 +3184,25 @@ movq 24(%rdi),%r11 movq 32(%rdi),%r12 movq %rdx,%r8 - imulq %rbx,%rdx + imulq %rbx,%rdx // n0*a[i] movq 40(%rdi),%r13 movq 48(%rdi),%r14 movq 56(%rdi),%r15 - movq %rax,24+8(%rsp) + movq %rax,24+8(%rsp) // store top-most carry bit leaq 64(%rdi),%rdi - xorq %rsi,%rsi + xorq %rsi,%rsi // cf=0,of=0 movq $-8,%rcx jmp L$sqrx8x_reduce .p2align 5 L$sqrx8x_reduce: movq %r8,%rbx - mulxq 0(%rbp),%rax,%r8 - adcxq %rbx,%rax + mulxq 0(%rbp),%rax,%r8 // n[0] + adcxq %rbx,%rax // discarded adoxq %r9,%r8 - mulxq 8(%rbp),%rbx,%r9 + mulxq 8(%rbp),%rbx,%r9 // n[1] adcxq %rbx,%r8 adoxq %r10,%r9 @@ -3214,15 +3214,15 @@ adcxq %rbx,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 +.byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 // mulx 8*4(%rbp),%rbx,%r12 movq %rdx,%rax movq %r8,%rdx adcxq %rbx,%r11 adoxq %r13,%r12 - mulxq 32+8(%rsp),%rbx,%rdx + mulxq 32+8(%rsp),%rbx,%rdx // %rdx discarded movq %rax,%rdx - movq %rax,64+48+8(%rsp,%rcx,8) + movq %rax,64+48+8(%rsp,%rcx,8) // put aside n0*a[i] mulxq 40(%rbp),%rax,%r13 adcxq %rax,%r12 @@ -3235,18 +3235,18 @@ mulxq 56(%rbp),%rax,%r15 movq %rbx,%rdx adcxq %rax,%r14 - adoxq %rsi,%r15 - adcxq %rsi,%r15 + adoxq %rsi,%r15 // %rsi is 0 + adcxq %rsi,%r15 // cf=0 .byte 0x67,0x67,0x67 - incq %rcx + incq %rcx // of=0 jnz L$sqrx8x_reduce - movq %rsi,%rax - cmpq 0+8(%rsp),%rbp + movq %rsi,%rax // xor %rax,%rax + cmpq 0+8(%rsp),%rbp // end of n[]? jae L$sqrx8x_no_tail - movq 48+8(%rsp),%rdx + movq 48+8(%rsp),%rdx // pull n0*a[0] addq 0(%rdi),%r8 leaq 64(%rbp),%rbp movq $-8,%rcx @@ -3258,9 +3258,9 @@ adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 leaq 64(%rdi),%rdi - sbbq %rax,%rax + sbbq %rax,%rax // top carry - xorq %rsi,%rsi + xorq %rsi,%rsi // of=0, cf=0 movq %rax,16+8(%rsp) jmp L$sqrx8x_tail @@ -3283,7 +3283,7 @@ adcxq %rax,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 +.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 // mulx 8*4(%rbp),%rax,%r12 adcxq %rax,%r11 adoxq %r13,%r12 @@ -3296,21 +3296,21 @@ adoxq %r15,%r14 mulxq 56(%rbp),%rax,%r15 - movq 72+48+8(%rsp,%rcx,8),%rdx + movq 72+48+8(%rsp,%rcx,8),%rdx // pull n0*a[i] adcxq %rax,%r14 adoxq %rsi,%r15 - movq %rbx,(%rdi,%rcx,8) + movq %rbx,(%rdi,%rcx,8) // save result movq %r8,%rbx - adcxq %rsi,%r15 + adcxq %rsi,%r15 // cf=0 - incq %rcx + incq %rcx // of=0 jnz L$sqrx8x_tail - cmpq 0+8(%rsp),%rbp - jae L$sqrx8x_tail_done + cmpq 0+8(%rsp),%rbp // end of n[]? + jae L$sqrx8x_tail_done // break out of loop - subq 16+8(%rsp),%rsi - movq 48+8(%rsp),%rdx + subq 16+8(%rsp),%rsi // mov 16(%rsp),%cf + movq 48+8(%rsp),%rdx // pull n0*a[0] leaq 64(%rbp),%rbp adcq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -3322,16 +3322,16 @@ adcq 56(%rdi),%r15 leaq 64(%rdi),%rdi sbbq %rax,%rax - subq $8,%rcx + subq $8,%rcx // mov $-8,%rcx - xorq %rsi,%rsi + xorq %rsi,%rsi // of=0, cf=0 movq %rax,16+8(%rsp) jmp L$sqrx8x_tail .p2align 5 L$sqrx8x_tail_done: xorq %rax,%rax - addq 24+8(%rsp),%r8 + addq 24+8(%rsp),%r8 // can this overflow? adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 @@ -3341,26 +3341,26 @@ adcq $0,%r15 adcq $0,%rax - subq 16+8(%rsp),%rsi -L$sqrx8x_no_tail: + subq 16+8(%rsp),%rsi // mov 16(%rsp),%cf +L$sqrx8x_no_tail: // %cf is 0 if jumped here adcq 0(%rdi),%r8 movq %xmm3,%rcx adcq 8(%rdi),%r9 movq 56(%rbp),%rsi - movq %xmm2,%rbp + movq %xmm2,%rbp // restore %rbp adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - adcq $0,%rax + adcq $0,%rax // top-most carry - movq 32+8(%rsp),%rbx - movq 64(%rdi,%rcx,1),%rdx + movq 32+8(%rsp),%rbx // n0 + movq 64(%rdi,%rcx,1),%rdx // modulo-scheduled "%r8" - movq %r8,0(%rdi) - leaq 64(%rdi),%r8 + movq %r8,0(%rdi) // store top 512 bits + leaq 64(%rdi),%r8 // borrow %r8 movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) @@ -3369,8 +3369,8 @@ movq %r14,48(%rdi) movq %r15,56(%rdi) - leaq 64(%rdi,%rcx,1),%rdi - cmpq 8+8(%rsp),%r8 + leaq 64(%rdi,%rcx,1),%rdi // start of current t[] window + cmpq 8+8(%rsp),%r8 // end of t[]? jb L$sqrx8x_reduction_loop ret @@ -3381,14 +3381,14 @@ __bn_postx4x_internal: movq 0(%rbp),%r12 - movq %rcx,%r10 - movq %rcx,%r9 + movq %rcx,%r10 // -%r9 + movq %rcx,%r9 // -%r9 negq %rax sarq $3+2,%rcx - - movq %xmm1,%rdx - movq %xmm1,%rsi - decq %r12 +// lea 48+8(%rsp,%r9),%rdi + movq %xmm1,%rdx // restore %rdx + movq %xmm1,%rsi // prepare for back-to-back call + decq %r12 // so that after 'not' we get -n[0] movq 8(%rbp),%r13 xorq %r8,%r8 movq 16(%rbp),%r14 @@ -3408,7 +3408,7 @@ andnq %rax,%r14,%r14 andnq %rax,%r15,%r15 - negq %r8 + negq %r8 // mov %r8,%cf adcq 0(%rdi),%r12 adcq 8(%rdi),%r13 adcq 16(%rdi),%r14 @@ -3416,7 +3416,7 @@ movq %r12,0(%rdx) leaq 32(%rdi),%rdi movq %r13,8(%rdx) - sbbq %r8,%r8 + sbbq %r8,%r8 // mov %cf,%r8 movq %r14,16(%rdx) movq %r15,24(%rdx) leaq 32(%rdx),%rdx @@ -3424,7 +3424,7 @@ incq %rcx jnz L$sqrx4x_sub - negq %r9 + negq %r9 // restore %r9 ret @@ -3440,13 +3440,13 @@ cmpl $0,%esi jz L$scatter_epilogue - - - - - - - +// %rdx stores 32 entries, t0 through t31. Each entry has %esi words. +// They are interleaved in memory as follows: +// +// t0[0] t1[0] t2[0] ... t31[0] +// t0[1] t1[1] t2[1] ... t31[1] +// ... +// t0[%esi-1] t1[%esi-1] t2[%esi-1] ... t31[%esi-1] leaq (%rdx,%rcx,8),%rdx L$scatter: @@ -3468,97 +3468,97 @@ .alt_entry _bn_gather5 _bn_gather5: -L$SEH_begin_bn_gather5: +L$SEH_begin_bn_gather5: // Win64 thing, but harmless in other cases _CET_ENDBR +// I can't trust assembler to use specific encoding:-( +.byte 0x4c,0x8d,0x14,0x24 // lea (%rsp),%r10 -.byte 0x4c,0x8d,0x14,0x24 - -.byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 +.byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 // sub $0x108,%rsp leaq L$inc(%rip),%rax - andq $-16,%rsp + andq $-16,%rsp // shouldn't be formally required movd %ecx,%xmm5 - movdqa 0(%rax),%xmm0 - movdqa 16(%rax),%xmm1 - leaq 128(%rdx),%r11 - leaq 128(%rsp),%rax + movdqa 0(%rax),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%rax),%xmm1 // 00000002000000020000000200000002 + leaq 128(%rdx),%r11 // size optimization + leaq 128(%rsp),%rax // size optimization - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast %ecx movdqa %xmm1,%xmm4 movdqa %xmm1,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,-128(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,-112(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,-96(%rax) movdqa %xmm4,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm3,-80(%rax) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,-64(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,-48(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,-32(%rax) movdqa %xmm4,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm3,-16(%rax) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,0(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,16(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,32(%rax) movdqa %xmm4,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm3,48(%rax) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,64(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,80(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,96(%rax) movdqa %xmm4,%xmm2 movdqa %xmm3,112(%rax) @@ -3618,10 +3618,10 @@ por %xmm3,%xmm5 por %xmm5,%xmm4 leaq 256(%r11),%r11 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 - movq %xmm0,(%rdi) + movq %xmm0,(%rdi) // m0=bp[0] leaq 8(%rdi),%rdi subl $1,%esi jnz L$gather
diff --git a/gen/bcm/x86_64-mont5-linux.S b/gen/bcm/x86_64-mont5-linux.S index 0a71a86..67fdfe0 100644 --- a/gen/bcm/x86_64-mont5-linux.S +++ b/gen/bcm/x86_64-mont5-linux.S
@@ -13,12 +13,12 @@ bn_mul_mont_gather5_nohw: .cfi_startproc _CET_ENDBR - - +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. Zero the upper half to normalize it. movl %r9d,%r9d movq %rsp,%rax .cfi_def_cfa_register %rax - movd 8(%rsp),%xmm5 + movd 8(%rsp),%xmm5 // load 7th argument pushq %rbx .cfi_offset %rbx,-16 pushq %rbp @@ -34,18 +34,18 @@ negq %r9 movq %rsp,%r11 - leaq -280(%rsp,%r9,8),%r10 - negq %r9 - andq $-1024,%r10 + leaq -280(%rsp,%r9,8),%r10 // future alloca(8*(num+2)+256+8) + negq %r9 // restore %r9 + andq $-1024,%r10 // minimize TLB usage - - - - - - - - +// An OS-agnostic version of __chkstk. +// +// Some OSes (Windows) insist on stack being "wired" to +// physical memory in strictly sequential manner, i.e. if stack +// allocation spans two pages, then reference to farmost one can +// be punishable by SEGV. But page walking can do good even on +// other OSes, because it guarantees that villain thread hits +// the guard page before it can make damage to innocent one... subq %r10,%r11 andq $-4096,%r11 leaq (%r10,%r11,1),%rsp @@ -62,35 +62,35 @@ .Lmul_page_walk_done: leaq .Linc(%rip),%r10 - movq %rax,8(%rsp,%r9,8) + movq %rax,8(%rsp,%r9,8) // tp[num+1]=%rsp .cfi_escape 0x0f,0x0a,0x77,0x08,0x79,0x00,0x38,0x1e,0x22,0x06,0x23,0x08 .Lmul_body: - leaq 128(%rdx),%r12 - movdqa 0(%r10),%xmm0 - movdqa 16(%r10),%xmm1 - leaq 24-112(%rsp,%r9,8),%r10 + leaq 128(%rdx),%r12 // reassign %rdx (+size optimization) + movdqa 0(%r10),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%r10),%xmm1 // 00000002000000020000000200000002 + leaq 24-112(%rsp,%r9,8),%r10 // place the mask after tp[num+3] (+ICache optimization) andq $-16,%r10 - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast index movdqa %xmm1,%xmm4 movdqa %xmm1,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 .byte 0x67 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,112(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,128(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,144(%r10) movdqa %xmm4,%xmm2 @@ -99,17 +99,17 @@ movdqa %xmm3,160(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,176(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,192(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,208(%r10) movdqa %xmm4,%xmm2 @@ -118,17 +118,17 @@ movdqa %xmm3,224(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,240(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,256(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,272(%r10) movdqa %xmm4,%xmm2 @@ -147,7 +147,7 @@ pcmpeqd %xmm5,%xmm3 movdqa %xmm2,336(%r10) - pand 64(%r12),%xmm0 + pand 64(%r12),%xmm0 // while it's still in register pand 80(%r12),%xmm1 pand 96(%r12),%xmm2 @@ -192,33 +192,33 @@ por %xmm2,%xmm0 por %xmm3,%xmm1 por %xmm1,%xmm0 - - pshufd $0x4e,%xmm0,%xmm1 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm0,%xmm1 // Swap upper and lower halves. por %xmm1,%xmm0 leaq 256(%r12),%r12 - movq %xmm0,%rbx + movq %xmm0,%rbx // m0=bp[0] - movq (%r8),%r8 + movq (%r8),%r8 // pull n0[0] value movq (%rsi),%rax - xorq %r14,%r14 - xorq %r15,%r15 + xorq %r14,%r14 // i=0 + xorq %r15,%r15 // j=0 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp .L1st_enter .align 16 @@ -226,30 +226,30 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] movq %r10,%r11 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 .L1st_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 - jne .L1st - + jne .L1st // note that upon exit %r15==%r9, so +// they can be used interchangeably addq %rax,%r13 adcq $0,%rdx - addq %r11,%r13 + addq %r11,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-16(%rsp,%r9,8) + movq %r13,-16(%rsp,%r9,8) // tp[num-1] movq %rdx,%r13 movq %r10,%r11 @@ -257,13 +257,13 @@ addq %r11,%r13 adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ jmp .Louter .align 16 .Louter: - leaq 24+128(%rsp,%r9,8),%rdx + leaq 24+128(%rsp,%r9,8),%rdx // where 256-byte mask is (+size optimization) andq $-16,%rdx pxor %xmm4,%xmm4 pxor %xmm5,%xmm5 @@ -316,34 +316,34 @@ por %xmm2,%xmm4 por %xmm3,%xmm5 por %xmm5,%xmm4 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 leaq 256(%r12),%r12 - movq (%rsi),%rax - movq %xmm0,%rbx + movq (%rsi),%rax // ap[0] + movq %xmm0,%rbx // m0=bp[i] - xorq %r15,%r15 + xorq %r15,%r15 // j=0 movq %r8,%rbp movq (%rsp),%r10 - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi),%rax adcq $0,%rdx - movq 8(%rsp),%r10 + movq 8(%rsp),%r10 // tp[1] movq %rdx,%r13 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ jmp .Linner_enter .align 16 @@ -351,78 +351,78 @@ addq %rax,%r13 movq (%rsi,%r15,8),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r15,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r15,8) + movq %r13,-16(%rsp,%r15,8) // tp[j-1] movq %rdx,%r13 .Linner_enter: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq (%rcx,%r15,8),%rax adcq $0,%rdx - addq %r11,%r10 + addq %r11,%r10 // ap[j]*bp[i]+tp[j] movq %rdx,%r11 adcq $0,%r11 - leaq 1(%r15),%r15 + leaq 1(%r15),%r15 // j++ - mulq %rbp + mulq %rbp // np[j]*m1 cmpq %r9,%r15 - jne .Linner - + jne .Linner // note that upon exit %r15==%r9, so +// they can be used interchangeably addq %rax,%r13 adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[i]+tp[j] movq (%rsp,%r9,8),%r10 adcq $0,%rdx - movq %r13,-16(%rsp,%r9,8) + movq %r13,-16(%rsp,%r9,8) // tp[num-1] movq %rdx,%r13 xorq %rdx,%rdx addq %r11,%r13 adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // pull upmost overflow bit adcq $0,%rdx movq %r13,-8(%rsp,%r9,8) - movq %rdx,(%rsp,%r9,8) + movq %rdx,(%rsp,%r9,8) // store upmost overflow bit - leaq 1(%r14),%r14 + leaq 1(%r14),%r14 // i++ cmpq %r9,%r14 jb .Louter - xorq %r14,%r14 - movq (%rsp),%rax - leaq (%rsp),%rsi - movq %r9,%r15 + xorq %r14,%r14 // i=0 and clear CF + movq (%rsp),%rax // tp[0] + leaq (%rsp),%rsi // borrow ap for tp + movq %r9,%r15 // j=num jmp .Lsub .align 16 .Lsub: sbbq (%rcx,%r14,8),%rax - movq %rax,(%rdi,%r14,8) - movq 8(%rsi,%r14,8),%rax - leaq 1(%r14),%r14 - decq %r15 + movq %rax,(%rdi,%r14,8) // rp[i]=tp[i]-np[i] + movq 8(%rsi,%r14,8),%rax // tp[i+1] + leaq 1(%r14),%r14 // i++ + decq %r15 // doesn't affect CF jnz .Lsub - sbbq $0,%rax + sbbq $0,%rax // handle upmost overflow bit movq $-1,%rbx xorq %rax,%rbx xorq %r14,%r14 - movq %r9,%r15 + movq %r9,%r15 // j=num -.Lcopy: +.Lcopy: // conditional copy movq (%rdi,%r14,8),%rcx movq (%rsp,%r14,8),%rdx andq %rbx,%rcx andq %rax,%rdx - movq %r14,(%rsp,%r14,8) + movq %r14,(%rsp,%r14,8) // zap temporary vector orq %rcx,%rdx - movq %rdx,(%rdi,%r14,8) + movq %rdx,(%rdi,%r14,8) // rp[i]=tp[i] leaq 1(%r14),%r14 subq $1,%r15 jnz .Lcopy - movq 8(%rsp,%r9,8),%rsi + movq 8(%rsp,%r9,8),%rsi // restore %rsp .cfi_def_cfa %rsi,8 movq $1,%rax @@ -469,36 +469,36 @@ .Lmul4x_prologue: .byte 0x67 +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, predates the first access. + shll $3,%r9d // convert %r9 to bytes + leaq (%r9,%r9,2),%r10 // 3*%r9 in bytes + negq %r9 // -%r9 - - - shll $3,%r9d - leaq (%r9,%r9,2),%r10 - negq %r9 - - - - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with +3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra [num] is allocated in order +// to align with bn_power5's frame, which is cleansed after +// completing exponentiation. Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb .Lmul4xsp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rdi + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) jmp .Lmul4xsp_done .align 32 .Lmul4xsp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -529,7 +529,7 @@ call mul4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp .cfi_def_cfa %rsi,8 movq $1,%rax @@ -556,36 +556,36 @@ .align 32 mul4x_internal: .cfi_startproc - shlq $5,%r9 - movd 8(%rax),%xmm5 + shlq $5,%r9 // %r9 was in bytes + movd 8(%rax),%xmm5 // load 7th argument, index leaq .Linc(%rip),%rax - leaq 128(%rdx,%r9,1),%r13 - shrq $5,%r9 - movdqa 0(%rax),%xmm0 - movdqa 16(%rax),%xmm1 - leaq 88-112(%rsp,%r9,1),%r10 - leaq 128(%rdx),%r12 + leaq 128(%rdx,%r9,1),%r13 // end of powers table (+size optimization) + shrq $5,%r9 // restore %r9 + movdqa 0(%rax),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%rax),%xmm1 // 00000002000000020000000200000002 + leaq 88-112(%rsp,%r9,1),%r10 // place the mask after tp[num+1] (+ICache optimization) + leaq 128(%rdx),%r12 // size optimization - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast index movdqa %xmm1,%xmm4 .byte 0x67,0x67 movdqa %xmm1,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 .byte 0x67 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,112(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,128(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,144(%r10) movdqa %xmm4,%xmm2 @@ -594,17 +594,17 @@ movdqa %xmm3,160(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,176(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,192(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,208(%r10) movdqa %xmm4,%xmm2 @@ -613,17 +613,17 @@ movdqa %xmm3,224(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,240(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,256(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,272(%r10) movdqa %xmm4,%xmm2 @@ -642,7 +642,7 @@ pcmpeqd %xmm5,%xmm3 movdqa %xmm2,336(%r10) - pand 64(%r12),%xmm0 + pand 64(%r12),%xmm0 // while it's still in register pand 80(%r12),%xmm1 pand 96(%r12),%xmm2 @@ -687,31 +687,31 @@ por %xmm2,%xmm0 por %xmm3,%xmm1 por %xmm1,%xmm0 - - pshufd $0x4e,%xmm0,%xmm1 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm0,%xmm1 // Swap upper and lower halves. por %xmm1,%xmm0 leaq 256(%r12),%r12 - movq %xmm0,%rbx + movq %xmm0,%rbx // m0=bp[0] - movq %r13,16+8(%rsp) - movq %rdi,56+8(%rsp) + movq %r13,16+8(%rsp) // save end of b[num] + movq %rdi,56+8(%rsp) // save %rdi - movq (%r8),%r8 + movq (%r8),%r8 // pull n0[0] value movq (%rsi),%rax - leaq (%rsi,%r9,1),%rsi + leaq (%rsi,%r9,1),%rsi // end of a[num] negq %r9 movq %r8,%rbp - mulq %rbx + mulq %rbx // ap[0]*bp[0] movq %rax,%r10 movq (%rcx),%rax - imulq %r10,%rbp + imulq %r10,%rbp // "tp[0]"*n0 leaq 64+8(%rsp),%r14 movq %rdx,%r11 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // discarded movq 8(%rsi,%r9,1),%rax adcq $0,%rdx movq %rdx,%rdi @@ -727,7 +727,7 @@ movq 16(%rsi,%r9,1),%rax adcq $0,%rdx addq %r11,%rdi - leaq 32(%r9),%r15 + leaq 32(%r9),%r15 // j=4 leaq 32(%rcx),%rcx adcq $0,%rdx movq %rdi,(%r14) @@ -736,103 +736,103 @@ .align 32 .L1st4x: - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx),%rax leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,1),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,1),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%r14) + movq %rdi,-16(%r14) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq 0(%rcx),%rax adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,1),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-8(%r14) + movq %r13,-8(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi,%r15,1),%rax adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] leaq 32(%rcx),%rcx adcq $0,%rdx - movq %rdi,(%r14) + movq %rdi,(%r14) // tp[j-1] movq %rdx,%r13 - addq $32,%r15 + addq $32,%r15 // j+=4 jnz .L1st4x - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r10 movq -16(%rcx),%rax leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi),%rax adcq $0,%rdx - addq %r10,%r13 + addq %r10,%r13 // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[0] addq %rax,%r11 movq -8(%rcx),%rax adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi,%r9,1),%rax + movq (%rsi,%r9,1),%rax // ap[0] adcq $0,%rdx - addq %r11,%rdi + addq %r11,%rdi // np[j]*m1+ap[j]*bp[0] adcq $0,%rdx - movq %rdi,-16(%r14) + movq %rdi,-16(%r14) // tp[j-1] movq %rdx,%r13 - leaq (%rcx,%r9,1),%rcx + leaq (%rcx,%r9,1),%rcx // rewind %rcx xorq %rdi,%rdi addq %r10,%r13 @@ -843,7 +843,7 @@ .align 32 .Louter4x: - leaq 16+128(%r14),%rdx + leaq 16+128(%r14),%rdx // where 256-byte mask is (+size optimization) pxor %xmm4,%xmm4 pxor %xmm5,%xmm5 movdqa -128(%r12),%xmm0 @@ -895,45 +895,45 @@ por %xmm2,%xmm4 por %xmm3,%xmm5 por %xmm5,%xmm4 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 leaq 256(%r12),%r12 - movq %xmm0,%rbx + movq %xmm0,%rbx // m0=bp[i] movq (%r14,%r9,1),%r10 movq %r8,%rbp - mulq %rbx - addq %rax,%r10 + mulq %rbx // ap[0]*bp[i] + addq %rax,%r10 // ap[0]*bp[i]+tp[0] movq (%rcx),%rax adcq $0,%rdx - imulq %r10,%rbp + imulq %r10,%rbp // tp[0]*n0 movq %rdx,%r11 - movq %rdi,(%r14) + movq %rdi,(%r14) // store upmost overflow bit - leaq (%r14,%r9,1),%r14 + leaq (%r14,%r9,1),%r14 // rewind %r14 - mulq %rbp - addq %rax,%r10 + mulq %rbp // np[0]*m1 + addq %rax,%r10 // "%r13", discarded movq 8(%rsi,%r9,1),%rax adcq $0,%rdx movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx - addq 8(%r14),%r11 + addq 8(%r14),%r11 // +tp[1] adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi,%r9,1),%rax adcq $0,%rdx - addq %r11,%rdi - leaq 32(%r9),%r15 + addq %r11,%rdi // np[j]*m1+ap[j]*bp[i]+tp[j] + leaq 32(%r9),%r15 // j=4 leaq 32(%rcx),%rcx adcq $0,%rdx movq %rdx,%r13 @@ -941,25 +941,25 @@ .align 32 .Linner4x: - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx),%rax adcq $0,%rdx - addq 16(%r14),%r10 + addq 16(%r14),%r10 // ap[j]*bp[i]+tp[j] leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi,%r15,1),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %rdi,-32(%r14) + movq %rdi,-32(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq -8(%rcx),%rax adcq $0,%rdx @@ -967,33 +967,33 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq (%rsi,%r15,1),%rax adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%r13 - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq 0(%rcx),%rax adcq $0,%rdx - addq (%r14),%r10 + addq (%r14),%r10 // ap[j]*bp[i]+tp[j] adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq 8(%rsi,%r15,1),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %rdi,-16(%r14) + movq %rdi,-16(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq 8(%rcx),%rax adcq $0,%rdx @@ -1001,38 +1001,38 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi movq 16(%rsi,%r15,1),%rax adcq $0,%rdx addq %r11,%rdi leaq 32(%rcx),%rcx adcq $0,%rdx - movq %r13,-8(%r14) + movq %r13,-8(%r14) // tp[j-1] movq %rdx,%r13 - addq $32,%r15 + addq $32,%r15 // j+=4 jnz .Linner4x - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r10 movq -16(%rcx),%rax adcq $0,%rdx - addq 16(%r14),%r10 + addq 16(%r14),%r10 // ap[j]*bp[i]+tp[j] leaq 32(%r14),%r14 adcq $0,%rdx movq %rdx,%r11 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%r13 movq -8(%rsi),%rax adcq $0,%rdx addq %r10,%r13 adcq $0,%rdx - movq %rdi,-32(%r14) + movq %rdi,-32(%r14) // tp[j-1] movq %rdx,%rdi - mulq %rbx + mulq %rbx // ap[j]*bp[i] addq %rax,%r11 movq %rbp,%rax movq -8(%rcx),%rbp @@ -1041,39 +1041,39 @@ adcq $0,%rdx movq %rdx,%r10 - mulq %rbp + mulq %rbp // np[j]*m1 addq %rax,%rdi - movq (%rsi,%r9,1),%rax + movq (%rsi,%r9,1),%rax // ap[0] adcq $0,%rdx addq %r11,%rdi adcq $0,%rdx - movq %r13,-24(%r14) + movq %r13,-24(%r14) // tp[j-1] movq %rdx,%r13 - movq %rdi,-16(%r14) - leaq (%rcx,%r9,1),%rcx + movq %rdi,-16(%r14) // tp[j-1] + leaq (%rcx,%r9,1),%rcx // rewind %rcx xorq %rdi,%rdi addq %r10,%r13 adcq $0,%rdi - addq (%r14),%r13 - adcq $0,%rdi + addq (%r14),%r13 // pull upmost overflow bit + adcq $0,%rdi // upmost overflow bit movq %r13,-8(%r14) cmpq 16+8(%rsp),%r12 jb .Louter4x xorq %rax,%rax - subq %r13,%rbp - adcq %r15,%r15 + subq %r13,%rbp // compare top-most words + adcq %r15,%r15 // %r15 is zero orq %r15,%rdi - subq %rdi,%rax - leaq (%r14,%r9,1),%rbx + subq %rdi,%rax // %rax=-%rdi + leaq (%r14,%r9,1),%rbx // tptr in .sqr4x_sub movq (%rcx),%r12 - leaq (%rcx),%rbp + leaq (%rcx),%rbp // nptr in .sqr4x_sub movq %r9,%rcx sarq $3+2,%rcx - movq 56+8(%rsp),%rdi - decq %r12 + movq 56+8(%rsp),%rdi // rptr in .sqr4x_sub + decq %r12 // so that after 'not' we get -n[0] xorq %r10,%r10 movq 8(%rbp),%r13 movq 16(%rbp),%r14 @@ -1104,35 +1104,35 @@ .cfi_offset %r15,-56 .Lpower5_prologue: - - - - shll $3,%r9d - leal (%r9,%r9,2),%r10d +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, come before the first access. + shll $3,%r9d // convert %r9 to bytes + leal (%r9,%r9,2),%r10d // 3*%r9 negq %r9 - movq (%r8),%r8 + movq (%r8),%r8 // *n0 - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with %rdi+3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb .Lpwr_sp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rsi + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) jmp .Lpwr_sp_done .align 32 .Lpwr_sp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*num*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -1158,22 +1158,22 @@ movq %r9,%r10 negq %r9 - - - - - - - - - +// ############################################################# +// Stack layout +// +// +0 saved %r9, used in reduction section +// +8 &t[2*%r9], used in reduction section +// +32 saved *n0 +// +40 saved %rsp +// +48 t[2*%r9] +// movq %r8,32(%rsp) - movq %rax,40(%rsp) + movq %rax,40(%rsp) // save original %rsp .cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08 .Lpower5_body: - movq %rdi,%xmm1 - movq %rcx,%xmm2 - movq %r10,%xmm3 + movq %rdi,%xmm1 // save %rdi, used in sqr8x + movq %rcx,%xmm2 // save %rcx + movq %r10,%xmm3 // -%r9, used in sqr8x movq %rdx,%xmm4 call __bn_sqr8x_internal @@ -1195,7 +1195,7 @@ call mul4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp .cfi_def_cfa %rsi,8 movq $1,%rax movq -48(%rsi),%r15 @@ -1226,261 +1226,261 @@ __bn_sqr8x_internal: .cfi_startproc _CET_ENDBR +// ############################################################# +// Squaring part: +// +// a) multiply-n-add everything but a[i]*a[i]; +// b) shift result of a) by 1 to the left and accumulate +// a[i]*a[i] products; +// +// ############################################################# +// a[1]a[0] +// a[2]a[0] +// a[3]a[0] +// a[2]a[1] +// a[4]a[0] +// a[3]a[1] +// a[5]a[0] +// a[4]a[1] +// a[3]a[2] +// a[6]a[0] +// a[5]a[1] +// a[4]a[2] +// a[7]a[0] +// a[6]a[1] +// a[5]a[2] +// a[4]a[3] +// a[7]a[1] +// a[6]a[2] +// a[5]a[3] +// a[7]a[2] +// a[6]a[3] +// a[5]a[4] +// a[7]a[3] +// a[6]a[4] +// a[7]a[4] +// a[6]a[5] +// a[7]a[5] +// a[7]a[6] +// a[1]a[0] +// a[2]a[0] +// a[3]a[0] +// a[4]a[0] +// a[5]a[0] +// a[6]a[0] +// a[7]a[0] +// a[2]a[1] +// a[3]a[1] +// a[4]a[1] +// a[5]a[1] +// a[6]a[1] +// a[7]a[1] +// a[3]a[2] +// a[4]a[2] +// a[5]a[2] +// a[6]a[2] +// a[7]a[2] +// a[4]a[3] +// a[5]a[3] +// a[6]a[3] +// a[7]a[3] +// a[5]a[4] +// a[6]a[4] +// a[7]a[4] +// a[6]a[5] +// a[7]a[5] +// a[7]a[6] +// a[0]a[0] +// a[1]a[1] +// a[2]a[2] +// a[3]a[3] +// a[4]a[4] +// a[5]a[5] +// a[6]a[6] +// a[7]a[7] + leaq 32(%r10),%rbp // %rbp=-(%r9-32) + leaq (%rsi,%r9,1),%rsi // end of a[] buffer, (%rsi,%rbp)=&ap[2] + movq %r9,%rcx // %rcx=%r9 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - leaq 32(%r10),%rbp - leaq (%rsi,%r9,1),%rsi - - movq %r9,%rcx - - - movq -32(%rsi,%rbp,1),%r14 - leaq 48+8(%rsp,%r9,2),%rdi - movq -24(%rsi,%rbp,1),%rax - leaq -32(%rdi,%rbp,1),%rdi - movq -16(%rsi,%rbp,1),%rbx +// comments apply to %r9==8 case + movq -32(%rsi,%rbp,1),%r14 // a[0] + leaq 48+8(%rsp,%r9,2),%rdi // end of tp[] buffer, &tp[2*%r9] + movq -24(%rsi,%rbp,1),%rax // a[1] + leaq -32(%rdi,%rbp,1),%rdi // end of tp[] window, &tp[2*%r9-"%rbp"] + movq -16(%rsi,%rbp,1),%rbx // a[2] movq %rax,%r15 - mulq %r14 - movq %rax,%r10 - movq %rbx,%rax + mulq %r14 // a[1]*a[0] + movq %rax,%r10 // a[1]*a[0] + movq %rbx,%rax // a[2] movq %rdx,%r11 - movq %r10,-24(%rdi,%rbp,1) + movq %r10,-24(%rdi,%rbp,1) // t[1] - mulq %r14 + mulq %r14 // a[2]*a[0] addq %rax,%r11 movq %rbx,%rax adcq $0,%rdx - movq %r11,-16(%rdi,%rbp,1) + movq %r11,-16(%rdi,%rbp,1) // t[2] movq %rdx,%r10 - movq -8(%rsi,%rbp,1),%rbx - mulq %r15 - movq %rax,%r12 + movq -8(%rsi,%rbp,1),%rbx // a[3] + mulq %r15 // a[2]*a[1] + movq %rax,%r12 // a[2]*a[1]+t[3] movq %rbx,%rax movq %rdx,%r13 leaq (%rbp),%rcx - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[3]*a[0] + addq %rax,%r10 // a[3]*a[0]+a[2]*a[1]+t[3] movq %rbx,%rax movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - movq %r10,-8(%rdi,%rcx,1) + movq %r10,-8(%rdi,%rcx,1) // t[3] jmp .Lsqr4x_1st .align 32 .Lsqr4x_1st: - movq (%rsi,%rcx,1),%rbx - mulq %r15 - addq %rax,%r13 + movq (%rsi,%rcx,1),%rbx // a[4] + mulq %r15 // a[3]*a[1] + addq %rax,%r13 // a[3]*a[1]+t[4] movq %rbx,%rax movq %rdx,%r12 adcq $0,%r12 - mulq %r14 - addq %rax,%r11 - movq %rbx,%rax - movq 8(%rsi,%rcx,1),%rbx + mulq %r14 // a[4]*a[0] + addq %rax,%r11 // a[4]*a[0]+a[3]*a[1]+t[4] + movq %rbx,%rax // a[3] + movq 8(%rsi,%rcx,1),%rbx // a[5] movq %rdx,%r10 adcq $0,%r10 addq %r13,%r11 adcq $0,%r10 - mulq %r15 - addq %rax,%r12 + mulq %r15 // a[4]*a[3] + addq %rax,%r12 // a[4]*a[3]+t[5] movq %rbx,%rax - movq %r11,(%rdi,%rcx,1) + movq %r11,(%rdi,%rcx,1) // t[4] movq %rdx,%r13 adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[5]*a[2] + addq %rax,%r10 // a[5]*a[2]+a[4]*a[3]+t[5] movq %rbx,%rax - movq 16(%rsi,%rcx,1),%rbx + movq 16(%rsi,%rcx,1),%rbx // a[6] movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - mulq %r15 - addq %rax,%r13 + mulq %r15 // a[5]*a[3] + addq %rax,%r13 // a[5]*a[3]+t[6] movq %rbx,%rax - movq %r10,8(%rdi,%rcx,1) + movq %r10,8(%rdi,%rcx,1) // t[5] movq %rdx,%r12 adcq $0,%r12 - mulq %r14 - addq %rax,%r11 - movq %rbx,%rax - movq 24(%rsi,%rcx,1),%rbx + mulq %r14 // a[6]*a[2] + addq %rax,%r11 // a[6]*a[2]+a[5]*a[3]+t[6] + movq %rbx,%rax // a[3] + movq 24(%rsi,%rcx,1),%rbx // a[7] movq %rdx,%r10 adcq $0,%r10 addq %r13,%r11 adcq $0,%r10 - mulq %r15 - addq %rax,%r12 + mulq %r15 // a[6]*a[5] + addq %rax,%r12 // a[6]*a[5]+t[7] movq %rbx,%rax - movq %r11,16(%rdi,%rcx,1) + movq %r11,16(%rdi,%rcx,1) // t[6] movq %rdx,%r13 adcq $0,%r13 leaq 32(%rcx),%rcx - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[7]*a[4] + addq %rax,%r10 // a[7]*a[4]+a[6]*a[5]+t[6] movq %rbx,%rax movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - movq %r10,-8(%rdi,%rcx,1) + movq %r10,-8(%rdi,%rcx,1) // t[7] cmpq $0,%rcx jne .Lsqr4x_1st - mulq %r15 + mulq %r15 // a[7]*a[5] addq %rax,%r13 leaq 16(%rbp),%rbp adcq $0,%rdx addq %r11,%r13 adcq $0,%rdx - movq %r13,(%rdi) + movq %r13,(%rdi) // t[8] movq %rdx,%r12 - movq %rdx,8(%rdi) + movq %rdx,8(%rdi) // t[9] jmp .Lsqr4x_outer .align 32 -.Lsqr4x_outer: - movq -32(%rsi,%rbp,1),%r14 - leaq 48+8(%rsp,%r9,2),%rdi - movq -24(%rsi,%rbp,1),%rax - leaq -32(%rdi,%rbp,1),%rdi - movq -16(%rsi,%rbp,1),%rbx +.Lsqr4x_outer: // comments apply to %r9==6 case + movq -32(%rsi,%rbp,1),%r14 // a[0] + leaq 48+8(%rsp,%r9,2),%rdi // end of tp[] buffer, &tp[2*%r9] + movq -24(%rsi,%rbp,1),%rax // a[1] + leaq -32(%rdi,%rbp,1),%rdi // end of tp[] window, &tp[2*%r9-"%rbp"] + movq -16(%rsi,%rbp,1),%rbx // a[2] movq %rax,%r15 - mulq %r14 - movq -24(%rdi,%rbp,1),%r10 - addq %rax,%r10 - movq %rbx,%rax + mulq %r14 // a[1]*a[0] + movq -24(%rdi,%rbp,1),%r10 // t[1] + addq %rax,%r10 // a[1]*a[0]+t[1] + movq %rbx,%rax // a[2] adcq $0,%rdx - movq %r10,-24(%rdi,%rbp,1) + movq %r10,-24(%rdi,%rbp,1) // t[1] movq %rdx,%r11 - mulq %r14 + mulq %r14 // a[2]*a[0] addq %rax,%r11 movq %rbx,%rax adcq $0,%rdx - addq -16(%rdi,%rbp,1),%r11 + addq -16(%rdi,%rbp,1),%r11 // a[2]*a[0]+t[2] movq %rdx,%r10 adcq $0,%r10 - movq %r11,-16(%rdi,%rbp,1) + movq %r11,-16(%rdi,%rbp,1) // t[2] xorq %r12,%r12 - movq -8(%rsi,%rbp,1),%rbx - mulq %r15 - addq %rax,%r12 + movq -8(%rsi,%rbp,1),%rbx // a[3] + mulq %r15 // a[2]*a[1] + addq %rax,%r12 // a[2]*a[1]+t[3] movq %rbx,%rax adcq $0,%rdx addq -8(%rdi,%rbp,1),%r12 movq %rdx,%r13 adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[3]*a[0] + addq %rax,%r10 // a[3]*a[0]+a[2]*a[1]+t[3] movq %rbx,%rax adcq $0,%rdx addq %r12,%r10 movq %rdx,%r11 adcq $0,%r11 - movq %r10,-8(%rdi,%rbp,1) + movq %r10,-8(%rdi,%rbp,1) // t[3] leaq (%rbp),%rcx jmp .Lsqr4x_inner .align 32 .Lsqr4x_inner: - movq (%rsi,%rcx,1),%rbx - mulq %r15 - addq %rax,%r13 + movq (%rsi,%rcx,1),%rbx // a[4] + mulq %r15 // a[3]*a[1] + addq %rax,%r13 // a[3]*a[1]+t[4] movq %rbx,%rax movq %rdx,%r12 adcq $0,%r12 @@ -1488,254 +1488,254 @@ adcq $0,%r12 .byte 0x67 - mulq %r14 - addq %rax,%r11 - movq %rbx,%rax - movq 8(%rsi,%rcx,1),%rbx + mulq %r14 // a[4]*a[0] + addq %rax,%r11 // a[4]*a[0]+a[3]*a[1]+t[4] + movq %rbx,%rax // a[3] + movq 8(%rsi,%rcx,1),%rbx // a[5] movq %rdx,%r10 adcq $0,%r10 addq %r13,%r11 adcq $0,%r10 - mulq %r15 - addq %rax,%r12 - movq %r11,(%rdi,%rcx,1) + mulq %r15 // a[4]*a[3] + addq %rax,%r12 // a[4]*a[3]+t[5] + movq %r11,(%rdi,%rcx,1) // t[4] movq %rbx,%rax movq %rdx,%r13 adcq $0,%r13 addq 8(%rdi,%rcx,1),%r12 - leaq 16(%rcx),%rcx + leaq 16(%rcx),%rcx // j++ adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[5]*a[2] + addq %rax,%r10 // a[5]*a[2]+a[4]*a[3]+t[5] movq %rbx,%rax adcq $0,%rdx addq %r12,%r10 movq %rdx,%r11 adcq $0,%r11 - movq %r10,-8(%rdi,%rcx,1) + movq %r10,-8(%rdi,%rcx,1) // t[5], "preloaded t[1]" below cmpq $0,%rcx jne .Lsqr4x_inner .byte 0x67 - mulq %r15 + mulq %r15 // a[5]*a[3] addq %rax,%r13 adcq $0,%rdx addq %r11,%r13 adcq $0,%rdx - movq %r13,(%rdi) + movq %r13,(%rdi) // t[6], "preloaded t[2]" below movq %rdx,%r12 - movq %rdx,8(%rdi) + movq %rdx,8(%rdi) // t[7], "preloaded t[3]" below addq $16,%rbp jnz .Lsqr4x_outer - - movq -32(%rsi),%r14 - leaq 48+8(%rsp,%r9,2),%rdi - movq -24(%rsi),%rax - leaq -32(%rdi,%rbp,1),%rdi - movq -16(%rsi),%rbx +// comments apply to %r9==4 case + movq -32(%rsi),%r14 // a[0] + leaq 48+8(%rsp,%r9,2),%rdi // end of tp[] buffer, &tp[2*%r9] + movq -24(%rsi),%rax // a[1] + leaq -32(%rdi,%rbp,1),%rdi // end of tp[] window, &tp[2*%r9-"%rbp"] + movq -16(%rsi),%rbx // a[2] movq %rax,%r15 - mulq %r14 - addq %rax,%r10 - movq %rbx,%rax + mulq %r14 // a[1]*a[0] + addq %rax,%r10 // a[1]*a[0]+t[1], preloaded t[1] + movq %rbx,%rax // a[2] movq %rdx,%r11 adcq $0,%r11 - mulq %r14 + mulq %r14 // a[2]*a[0] addq %rax,%r11 movq %rbx,%rax - movq %r10,-24(%rdi) + movq %r10,-24(%rdi) // t[1] movq %rdx,%r10 adcq $0,%r10 - addq %r13,%r11 - movq -8(%rsi),%rbx + addq %r13,%r11 // a[2]*a[0]+t[2], preloaded t[2] + movq -8(%rsi),%rbx // a[3] adcq $0,%r10 - mulq %r15 - addq %rax,%r12 + mulq %r15 // a[2]*a[1] + addq %rax,%r12 // a[2]*a[1]+t[3], preloaded t[3] movq %rbx,%rax - movq %r11,-16(%rdi) + movq %r11,-16(%rdi) // t[2] movq %rdx,%r13 adcq $0,%r13 - mulq %r14 - addq %rax,%r10 + mulq %r14 // a[3]*a[0] + addq %rax,%r10 // a[3]*a[0]+a[2]*a[1]+t[3] movq %rbx,%rax movq %rdx,%r11 adcq $0,%r11 addq %r12,%r10 adcq $0,%r11 - movq %r10,-8(%rdi) + movq %r10,-8(%rdi) // t[3] - mulq %r15 + mulq %r15 // a[3]*a[1] addq %rax,%r13 - movq -16(%rsi),%rax + movq -16(%rsi),%rax // a[2] adcq $0,%rdx addq %r11,%r13 adcq $0,%rdx - movq %r13,(%rdi) + movq %r13,(%rdi) // t[4] movq %rdx,%r12 - movq %rdx,8(%rdi) + movq %rdx,8(%rdi) // t[5] - mulq %rbx + mulq %rbx // a[2]*a[3] addq $16,%rbp xorq %r14,%r14 - subq %r9,%rbp + subq %r9,%rbp // %rbp=16-%r9 xorq %r15,%r15 - addq %r12,%rax + addq %r12,%rax // t[5] adcq $0,%rdx - movq %rax,8(%rdi) - movq %rdx,16(%rdi) - movq %r15,24(%rdi) + movq %rax,8(%rdi) // t[5] + movq %rdx,16(%rdi) // t[6] + movq %r15,24(%rdi) // t[7] - movq -16(%rsi,%rbp,1),%rax + movq -16(%rsi,%rbp,1),%rax // a[0] leaq 48+8(%rsp),%rdi - xorq %r10,%r10 - movq 8(%rdi),%r11 + xorq %r10,%r10 // t[0] + movq 8(%rdi),%r11 // t[1] - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq 16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 24(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq 16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 24(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq -8(%rsi,%rbp,1),%rax + movq -8(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %r12,(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1 | shift movq %r13,8(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - movq 32(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 40(%rdi),%r11 + orq %r10,%r8 // | t[2*i]>>63 + movq 32(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 40(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%rbx - movq 0(%rsi,%rbp,1),%rax + movq 0(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %rbx,16(%rdi) adcq %rdx,%r8 leaq 16(%rbp),%rbp movq %r8,24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 leaq 64(%rdi),%rdi jmp .Lsqr4x_shift_n_add .align 32 .Lsqr4x_shift_n_add: - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq -16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq -8(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq -16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq -8(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq -8(%rsi,%rbp,1),%rax + movq -8(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %r12,-32(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1 | shift movq %r13,-24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - movq 0(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 8(%rdi),%r11 + orq %r10,%r8 // | t[2*i]>>63 + movq 0(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 8(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%rbx - movq 0(%rsi,%rbp,1),%rax + movq 0(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %rbx,-16(%rdi) adcq %rdx,%r8 - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift movq %r8,-8(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq 16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 24(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq 16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 24(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq 8(%rsi,%rbp,1),%rax + movq 8(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %r12,0(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1 | shift movq %r13,8(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - movq 32(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq 40(%rdi),%r11 + orq %r10,%r8 // | t[2*i]>>63 + movq 32(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq 40(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%rbx - movq 16(%rsi,%rbp,1),%rax + movq 16(%rsi,%rbp,1),%rax // a[i+1] # prefetch movq %rbx,16(%rdi) adcq %rdx,%r8 movq %r8,24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 leaq 64(%rdi),%rdi addq $32,%rbp jnz .Lsqr4x_shift_n_add - leaq (%r14,%r10,2),%r12 + leaq (%r14,%r10,2),%r12 // t[2*i]<<1 | shift .byte 0x67 shrq $63,%r10 - leaq (%rcx,%r11,2),%r13 + leaq (%rcx,%r11,2),%r13 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r13 - movq -16(%rdi),%r10 - movq %r11,%r14 - mulq %rax - negq %r15 - movq -8(%rdi),%r11 + orq %r10,%r13 // | t[2*i]>>63 + movq -16(%rdi),%r10 // t[2*i+2] # prefetch + movq %r11,%r14 // shift=t[2*i+1]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf + movq -8(%rdi),%r11 // t[2*i+2+1] # prefetch adcq %rax,%r12 - movq -8(%rsi),%rax + movq -8(%rsi),%rax // a[i+1] # prefetch movq %r12,-32(%rdi) adcq %rdx,%r13 - leaq (%r14,%r10,2),%rbx + leaq (%r14,%r10,2),%rbx // t[2*i]<<1|shift movq %r13,-24(%rdi) - sbbq %r15,%r15 + sbbq %r15,%r15 // mov cf,%r15 shrq $63,%r10 - leaq (%rcx,%r11,2),%r8 + leaq (%rcx,%r11,2),%r8 // t[2*i+1]<<1 | shrq $63,%r11 - orq %r10,%r8 - mulq %rax - negq %r15 + orq %r10,%r8 // | t[2*i]>>63 + mulq %rax // a[i]*a[i] + negq %r15 // mov %r15,cf adcq %rax,%rbx adcq %rdx,%r8 movq %rbx,-16(%rdi) @@ -1743,17 +1743,17 @@ movq %xmm2,%rbp __bn_sqr8x_reduction: xorq %rax,%rax - leaq (%r9,%rbp,1),%rcx - leaq 48+8(%rsp,%r9,2),%rdx + leaq (%r9,%rbp,1),%rcx // end of n[] + leaq 48+8(%rsp,%r9,2),%rdx // end of t[] buffer movq %rcx,0+8(%rsp) - leaq 48+8(%rsp,%r9,1),%rdi + leaq 48+8(%rsp,%r9,1),%rdi // end of initial t[] window movq %rdx,8+8(%rsp) negq %r9 jmp .L8x_reduction_loop .align 32 .L8x_reduction_loop: - leaq (%rdi,%r9,1),%rdi + leaq (%rdi,%r9,1),%rdi // start of current t[] window .byte 0x66 movq 0(%rdi),%rbx movq 8(%rdi),%r9 @@ -1763,20 +1763,20 @@ movq 40(%rdi),%r13 movq 48(%rdi),%r14 movq 56(%rdi),%r15 - movq %rax,(%rdx) + movq %rax,(%rdx) // store top-most carry bit leaq 64(%rdi),%rdi .byte 0x67 movq %rbx,%r8 - imulq 32+8(%rsp),%rbx - movq 0(%rbp),%rax + imulq 32+8(%rsp),%rbx // n0*a[0] + movq 0(%rbp),%rax // n[0] movl $8,%ecx jmp .L8x_reduce .align 32 .L8x_reduce: mulq %rbx - movq 8(%rbp),%rax + movq 8(%rbp),%rax // n[1] negq %r8 movq %rdx,%r8 adcq $0,%r8 @@ -1786,7 +1786,7 @@ movq 16(%rbp),%rax adcq $0,%rdx addq %r9,%r8 - movq %rbx,48-8+8(%rsp,%rcx,8) + movq %rbx,48-8+8(%rsp,%rcx,8) // put aside n0*a[i] movq %rdx,%r9 adcq $0,%r9 @@ -1795,7 +1795,7 @@ movq 24(%rbp),%rax adcq $0,%rdx addq %r10,%r9 - movq 32+8(%rsp),%rsi + movq 32+8(%rsp),%rsi // pull n0, borrow %rsi movq %rdx,%r10 adcq $0,%r10 @@ -1803,7 +1803,7 @@ addq %rax,%r11 movq 32(%rbp),%rax adcq $0,%rdx - imulq %r8,%rsi + imulq %r8,%rsi // modulo-scheduled addq %r11,%r10 movq %rdx,%r11 adcq $0,%r11 @@ -1833,9 +1833,9 @@ adcq $0,%r14 mulq %rbx - movq %rsi,%rbx + movq %rsi,%rbx // n0*a[i] addq %rax,%r15 - movq 0(%rbp),%rax + movq 0(%rbp),%rax // n[0] adcq $0,%rdx addq %r15,%r14 movq %rdx,%r15 @@ -1846,8 +1846,8 @@ leaq 64(%rbp),%rbp xorq %rax,%rax - movq 8+8(%rsp),%rdx - cmpq 0+8(%rsp),%rbp + movq 8+8(%rsp),%rdx // pull end of t[] + cmpq 0+8(%rsp),%rbp // end of n[]? jae .L8x_no_tail .byte 0x66 @@ -1859,9 +1859,9 @@ adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - sbbq %rsi,%rsi + sbbq %rsi,%rsi // top carry - movq 48+56+8(%rsp),%rbx + movq 48+56+8(%rsp),%rbx // pull n0*a[0] movl $8,%ecx movq 0(%rbp),%rax jmp .L8x_tail @@ -1871,7 +1871,7 @@ mulq %rbx addq %rax,%r8 movq 8(%rbp),%rax - movq %r8,(%rdi) + movq %r8,(%rdi) // save result movq %rdx,%r8 adcq $0,%r8 @@ -1880,7 +1880,7 @@ movq 16(%rbp),%rax adcq $0,%rdx addq %r9,%r8 - leaq 8(%rdi),%rdi + leaq 8(%rdi),%rdi // %rdi++ movq %rdx,%r9 adcq $0,%r9 @@ -1925,11 +1925,11 @@ adcq $0,%r14 mulq %rbx - movq 48-16+8(%rsp,%rcx,8),%rbx + movq 48-16+8(%rsp,%rcx,8),%rbx // pull n0*a[i] addq %rax,%r15 adcq $0,%rdx addq %r15,%r14 - movq 0(%rbp),%rax + movq 0(%rbp),%rax // pull n[0] movq %rdx,%r15 adcq $0,%r15 @@ -1937,13 +1937,13 @@ jnz .L8x_tail leaq 64(%rbp),%rbp - movq 8+8(%rsp),%rdx - cmpq 0+8(%rsp),%rbp - jae .L8x_tail_done + movq 8+8(%rsp),%rdx // pull end of t[] + cmpq 0+8(%rsp),%rbp // end of n[]? + jae .L8x_tail_done // break out of loop - movq 48+56+8(%rsp),%rbx + movq 48+56+8(%rsp),%rbx // pull n0*a[0] negq %rsi - movq 0(%rbp),%rax + movq 0(%rbp),%rax // pull n[0] adcq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 @@ -1952,7 +1952,7 @@ adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - sbbq %rsi,%rsi + sbbq %rsi,%rsi // top carry movl $8,%ecx jmp .L8x_tail @@ -1960,7 +1960,7 @@ .align 32 .L8x_tail_done: xorq %rax,%rax - addq (%rdx),%r8 + addq (%rdx),%r8 // can this overflow? adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 @@ -1980,15 +1980,15 @@ adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - adcq $0,%rax - movq -8(%rbp),%rcx + adcq $0,%rax // top-most carry + movq -8(%rbp),%rcx // np[num-1] xorq %rsi,%rsi - movq %xmm2,%rbp + movq %xmm2,%rbp // restore %rbp - movq %r8,0(%rdi) + movq %r8,0(%rdi) // store top 512 bits movq %r9,8(%rdi) - movq %xmm3,%r9 + movq %xmm3,%r9 // %r9 is %r9, can't be moved upwards movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) @@ -1997,7 +1997,7 @@ movq %r15,56(%rdi) leaq 64(%rdi),%rdi - cmpq %rdx,%rdi + cmpq %rdx,%rdi // end of t[]? jb .L8x_reduction_loop ret .cfi_endproc @@ -2007,13 +2007,13 @@ __bn_post4x_internal: .cfi_startproc movq 0(%rbp),%r12 - leaq (%rdi,%r9,1),%rbx + leaq (%rdi,%r9,1),%rbx // %rdi was %rbx above movq %r9,%rcx - movq %xmm1,%rdi + movq %xmm1,%rdi // restore %rdi negq %rax - movq %xmm1,%rsi + movq %xmm1,%rsi // prepare for back-to-back call sarq $3+2,%rcx - decq %r12 + decq %r12 // so that after 'not' we get -n[0] xorq %r10,%r10 movq 8(%rbp),%r13 movq 16(%rbp),%r14 @@ -2037,7 +2037,7 @@ andq %rax,%r14 andq %rax,%r15 - negq %r10 + negq %r10 // mov %r10,%cf adcq 0(%rbx),%r12 adcq 8(%rbx),%r13 adcq 16(%rbx),%r14 @@ -2045,16 +2045,16 @@ movq %r12,0(%rdi) leaq 32(%rbx),%rbx movq %r13,8(%rdi) - sbbq %r10,%r10 + sbbq %r10,%r10 // mov %cf,%r10 movq %r14,16(%rdi) movq %r15,24(%rdi) leaq 32(%rdi),%rdi - incq %rcx + incq %rcx // pass %cf jnz .Lsqr4x_sub - movq %r9,%r10 - negq %r9 + movq %r9,%r10 // prepare for back-to-back call + negq %r9 // restore %r9 ret .cfi_endproc .size __bn_post4x_internal,.-__bn_post4x_internal @@ -2081,42 +2081,42 @@ .cfi_offset %r15,-56 .Lmulx4x_prologue: +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, predates the first access. + shll $3,%r9d // convert %r9 to bytes + leaq (%r9,%r9,2),%r10 // 3*%r9 in bytes + negq %r9 // -%r9 + movq (%r8),%r8 // *n0 - - - shll $3,%r9d - leaq (%r9,%r9,2),%r10 - negq %r9 - movq (%r8),%r8 - - - - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with +3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra [num] is allocated in order +// to align with bn_power5's frame, which is cleansed after +// completing exponentiation. Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb .Lmulx4xsp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9*8+256) jmp .Lmulx4xsp_done .Lmulx4xsp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 subq %r11,%rbp .Lmulx4xsp_done: - andq $-64,%rbp + andq $-64,%rbp // ensure alignment movq %rsp,%r11 subq %rbp,%r11 andq $-4096,%r11 @@ -2133,25 +2133,25 @@ ja .Lmulx4x_page_walk .Lmulx4x_page_walk_done: - - - - - - - - - - - - - movq %r8,32(%rsp) - movq %rax,40(%rsp) +// ############################################################# +// Stack layout +// +0 -num +// +8 off-loaded &b[i] +// +16 end of b[num] +// +24 inner counter +// +32 saved n0 +// +40 saved %rsp +// +48 +// +56 saved rp +// +64 tmp[num+1] +// + movq %r8,32(%rsp) // save *n0 + movq %rax,40(%rsp) // save original %rsp .cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08 .Lmulx4x_body: call mulx4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp .cfi_def_cfa %rsi,8 movq $1,%rax @@ -2178,44 +2178,44 @@ .align 32 mulx4x_internal: .cfi_startproc - movq %r9,8(%rsp) + movq %r9,8(%rsp) // save -%r9 (it was in bytes) movq %r9,%r10 - negq %r9 + negq %r9 // restore %r9 shlq $5,%r9 - negq %r10 - leaq 128(%rdx,%r9,1),%r13 + negq %r10 // restore %r9 + leaq 128(%rdx,%r9,1),%r13 // end of powers table (+size optimization) shrq $5+5,%r9 - movd 8(%rax),%xmm5 + movd 8(%rax),%xmm5 // load 7th argument subq $1,%r9 leaq .Linc(%rip),%rax - movq %r13,16+8(%rsp) - movq %r9,24+8(%rsp) - movq %rdi,56+8(%rsp) - movdqa 0(%rax),%xmm0 - movdqa 16(%rax),%xmm1 - leaq 88-112(%rsp,%r10,1),%r10 - leaq 128(%rdx),%rdi + movq %r13,16+8(%rsp) // end of b[num] + movq %r9,24+8(%rsp) // inner counter + movq %rdi,56+8(%rsp) // save %rdi + movdqa 0(%rax),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%rax),%xmm1 // 00000002000000020000000200000002 + leaq 88-112(%rsp,%r10,1),%r10 // place the mask after tp[num+1] (+ICache optimization) + leaq 128(%rdx),%rdi // size optimization - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast index movdqa %xmm1,%xmm4 .byte 0x67 movdqa %xmm1,%xmm2 .byte 0x67 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,112(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,128(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,144(%r10) movdqa %xmm4,%xmm2 @@ -2224,17 +2224,17 @@ movdqa %xmm3,160(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,176(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,192(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,208(%r10) movdqa %xmm4,%xmm2 @@ -2243,17 +2243,17 @@ movdqa %xmm3,224(%r10) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,240(%r10) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,256(%r10) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,272(%r10) movdqa %xmm4,%xmm2 @@ -2273,7 +2273,7 @@ pcmpeqd %xmm5,%xmm3 movdqa %xmm2,336(%r10) - pand 64(%rdi),%xmm0 + pand 64(%rdi),%xmm0 // while it's still in register pand 80(%rdi),%xmm1 pand 96(%rdi),%xmm2 movdqa %xmm3,352(%r10) @@ -2317,41 +2317,41 @@ por %xmm2,%xmm0 por %xmm3,%xmm1 pxor %xmm1,%xmm0 - - pshufd $0x4e,%xmm0,%xmm1 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm0,%xmm1 // Swap upper and lower halves. por %xmm1,%xmm0 leaq 256(%rdi),%rdi - movq %xmm0,%rdx + movq %xmm0,%rdx // bp[0] leaq 64+32+8(%rsp),%rbx movq %rdx,%r9 - mulxq 0(%rsi),%r8,%rax - mulxq 8(%rsi),%r11,%r12 + mulxq 0(%rsi),%r8,%rax // a[0]*b[0] + mulxq 8(%rsi),%r11,%r12 // a[1]*b[0] addq %rax,%r11 - mulxq 16(%rsi),%rax,%r13 + mulxq 16(%rsi),%rax,%r13 // ... adcq %rax,%r12 adcq $0,%r13 mulxq 24(%rsi),%rax,%r14 movq %r8,%r15 - imulq 32+8(%rsp),%r8 - xorq %rbp,%rbp + imulq 32+8(%rsp),%r8 // "t[0]"*n0 + xorq %rbp,%rbp // cf=0, of=0 movq %r8,%rdx - movq %rdi,8+8(%rsp) + movq %rdi,8+8(%rsp) // off-load &b[i] leaq 32(%rsi),%rsi adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%r15 + adcxq %rax,%r15 // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 adoxq %r12,%r11 mulxq 16(%rcx),%rax,%r12 - movq 24+8(%rsp),%rdi + movq 24+8(%rsp),%rdi // counter value movq %r10,-32(%rbx) adcxq %rax,%r11 adoxq %r13,%r12 @@ -2359,25 +2359,25 @@ movq %r9,%rdx movq %r11,-24(%rbx) adcxq %rax,%r12 - adoxq %rbp,%r15 + adoxq %rbp,%r15 // of=0 leaq 32(%rcx),%rcx movq %r12,-16(%rbx) jmp .Lmulx4x_1st .align 32 .Lmulx4x_1st: - adcxq %rbp,%r15 - mulxq 0(%rsi),%r10,%rax + adcxq %rbp,%r15 // cf=0, modulo-scheduled + mulxq 0(%rsi),%r10,%rax // a[4]*b[0] adcxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[0] adcxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 .byte 0x67,0x67 movq %r8,%rdx adcxq %rax,%r13 - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx @@ -2401,21 +2401,21 @@ leaq 32(%rcx),%rcx movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz .Lmulx4x_1st - movq 8(%rsp),%rax - adcq %rbp,%r15 - leaq (%rsi,%rax,1),%rsi + movq 8(%rsp),%rax // load -num + adcq %rbp,%r15 // modulo-scheduled + leaq (%rsi,%rax,1),%rsi // rewind %rsi addq %r15,%r14 - movq 8+8(%rsp),%rdi - adcq %rbp,%rbp + movq 8+8(%rsp),%rdi // re-load &b[i] + adcq %rbp,%rbp // top-most carry movq %r14,-8(%rbx) jmp .Lmulx4x_outer .align 32 .Lmulx4x_outer: - leaq 16-256(%rbx),%r10 + leaq 16-256(%rbx),%r10 // where 256-byte mask is (+density control) pxor %xmm4,%xmm4 .byte 0x67,0x67 pxor %xmm5,%xmm5 @@ -2468,41 +2468,41 @@ por %xmm2,%xmm4 por %xmm3,%xmm5 por %xmm5,%xmm4 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 leaq 256(%rdi),%rdi - movq %xmm0,%rdx + movq %xmm0,%rdx // m0=bp[i] - movq %rbp,(%rbx) - leaq 32(%rbx,%rax,1),%rbx - mulxq 0(%rsi),%r8,%r11 - xorq %rbp,%rbp + movq %rbp,(%rbx) // save top-most carry + leaq 32(%rbx,%rax,1),%rbx // rewind %rbx + mulxq 0(%rsi),%r8,%r11 // a[0]*b[i] + xorq %rbp,%rbp // cf=0, of=0 movq %rdx,%r9 - mulxq 8(%rsi),%r14,%r12 - adoxq -32(%rbx),%r8 + mulxq 8(%rsi),%r14,%r12 // a[1]*b[i] + adoxq -32(%rbx),%r8 // +t[0] adcxq %r14,%r11 - mulxq 16(%rsi),%r15,%r13 + mulxq 16(%rsi),%r15,%r13 // ... adoxq -24(%rbx),%r11 adcxq %r15,%r12 mulxq 24(%rsi),%rdx,%r14 adoxq -16(%rbx),%r12 adcxq %rdx,%r13 - leaq (%rcx,%rax,1),%rcx + leaq (%rcx,%rax,1),%rcx // rewind %rcx leaq 32(%rsi),%rsi adoxq -8(%rbx),%r13 adcxq %rbp,%r14 adoxq %rbp,%r14 movq %r8,%r15 - imulq 32+8(%rsp),%r8 + imulq 32+8(%rsp),%r8 // "t[0]"*n0 movq %r8,%rdx - xorq %rbp,%rbp - movq %rdi,8+8(%rsp) + xorq %rbp,%rbp // cf=0, of=0 + movq %rdi,8+8(%rsp) // off-load &b[i] mulxq 0(%rcx),%rax,%r10 - adcxq %rax,%r15 + adcxq %rax,%r15 // discarded adoxq %r11,%r10 mulxq 8(%rcx),%rax,%r11 adcxq %rax,%r10 @@ -2512,24 +2512,24 @@ adoxq %r13,%r12 mulxq 24(%rcx),%rax,%r15 movq %r9,%rdx - movq 24+8(%rsp),%rdi + movq 24+8(%rsp),%rdi // counter value movq %r10,-32(%rbx) adcxq %rax,%r12 movq %r11,-24(%rbx) - adoxq %rbp,%r15 + adoxq %rbp,%r15 // of=0 movq %r12,-16(%rbx) leaq 32(%rcx),%rcx jmp .Lmulx4x_inner .align 32 .Lmulx4x_inner: - mulxq 0(%rsi),%r10,%rax - adcxq %rbp,%r15 + mulxq 0(%rsi),%r10,%rax // a[4]*b[i] + adcxq %rbp,%r15 // cf=0, modulo-scheduled adoxq %r14,%r10 - mulxq 8(%rsi),%r11,%r14 + mulxq 8(%rsi),%r11,%r14 // a[5]*b[i] adcxq 0(%rbx),%r10 adoxq %rax,%r11 - mulxq 16(%rsi),%r12,%rax + mulxq 16(%rsi),%r12,%rax // ... adcxq 8(%rbx),%r11 adoxq %r14,%r12 mulxq 24(%rsi),%r13,%r14 @@ -2537,10 +2537,10 @@ adcxq 16(%rbx),%r12 adoxq %rax,%r13 adcxq 24(%rbx),%r13 - adoxq %rbp,%r14 + adoxq %rbp,%r14 // of=0 leaq 32(%rsi),%rsi leaq 32(%rbx),%rbx - adcxq %rbp,%r14 + adcxq %rbp,%r14 // cf=0 adoxq %r15,%r10 mulxq 0(%rcx),%rax,%r15 @@ -2562,17 +2562,17 @@ adoxq %rbp,%r15 movq %r13,-16(%rbx) - decq %rdi + decq %rdi // of=0, pass cf jnz .Lmulx4x_inner - movq 0+8(%rsp),%rax - adcq %rbp,%r15 - subq 0(%rbx),%rdi - movq 8+8(%rsp),%rdi + movq 0+8(%rsp),%rax // load -num + adcq %rbp,%r15 // modulo-scheduled + subq 0(%rbx),%rdi // pull top-most carry to %cf + movq 8+8(%rsp),%rdi // re-load &b[i] movq 16+8(%rsp),%r10 adcq %r15,%r14 - leaq (%rsi,%rax,1),%rsi - adcq %rbp,%rbp + leaq (%rsi,%rax,1),%rsi // rewind %rsi + adcq %rbp,%rbp // top-most carry movq %r14,-8(%rbx) cmpq %r10,%rdi @@ -2581,23 +2581,23 @@ movq -8(%rcx),%r10 movq %rbp,%r8 movq (%rcx,%rax,1),%r12 - leaq (%rcx,%rax,1),%rbp + leaq (%rcx,%rax,1),%rbp // rewind %rcx movq %rax,%rcx - leaq (%rbx,%rax,1),%rdi + leaq (%rbx,%rax,1),%rdi // rewind %rbx xorl %eax,%eax xorq %r15,%r15 - subq %r14,%r10 + subq %r14,%r10 // compare top-most words adcq %r15,%r15 orq %r15,%r8 sarq $3+2,%rcx - subq %r8,%rax - movq 56+8(%rsp),%rdx - decq %r12 + subq %r8,%rax // %rax=-%r8 + movq 56+8(%rsp),%rdx // restore rp + decq %r12 // so that after 'not' we get -n[0] movq 8(%rbp),%r13 xorq %r8,%r8 movq 16(%rbp),%r14 movq 24(%rbp),%r15 - jmp .Lsqrx4x_sub_entry + jmp .Lsqrx4x_sub_entry // common post-condition .cfi_endproc .size mulx4x_internal,.-mulx4x_internal .globl bn_powerx5 @@ -2623,35 +2623,35 @@ .cfi_offset %r15,-56 .Lpowerx5_prologue: - - - - shll $3,%r9d - leaq (%r9,%r9,2),%r10 +// num is declared as an int, a 32-bit parameter, so the upper half is +// undefined. It is important that this write to %r9, which zeros the +// upper half, predates the first access. + shll $3,%r9d // convert %r9 to bytes + leaq (%r9,%r9,2),%r10 // 3*%r9 in bytes negq %r9 - movq (%r8),%r8 + movq (%r8),%r8 // *n0 - - - - - - - +// ############################################################# +// Ensure that stack frame doesn't alias with %rdi+3*%r9 +// modulo 4096, which covers ret[num], am[num] and n[num] +// (see bn_exp.c). This is done to allow memory disambiguation +// logic do its magic. [Extra 256 bytes is for power mask +// calculated from 7th argument, the index.] +// leaq -320(%rsp,%r9,2),%r11 movq %rsp,%rbp subq %rdi,%r11 andq $4095,%r11 cmpq %r11,%r10 jb .Lpwrx_sp_alt - subq %r11,%rbp - leaq -320(%rbp,%r9,2),%rbp + subq %r11,%rbp // align with %rsi + leaq -320(%rbp,%r9,2),%rbp // future alloca(frame+2*%r9*8+256) jmp .Lpwrx_sp_done .align 32 .Lpwrx_sp_alt: leaq 4096-320(,%r9,2),%r10 - leaq -320(%rbp,%r9,2),%rbp + leaq -320(%rbp,%r9,2),%rbp // alloca(frame+2*%r9*8+256) subq %r10,%r11 movq $0,%r10 cmovcq %r10,%r11 @@ -2677,24 +2677,24 @@ movq %r9,%r10 negq %r9 - - - - - - - - - - - +// ############################################################# +// Stack layout +// +// +0 saved %r9, used in reduction section +// +8 &t[2*%r9], used in reduction section +// +16 intermediate carry bit +// +24 top-most carry bit, used in reduction section +// +32 saved *n0 +// +40 saved %rsp +// +48 t[2*%r9] +// pxor %xmm0,%xmm0 - movq %rdi,%xmm1 - movq %rcx,%xmm2 - movq %r10,%xmm3 + movq %rdi,%xmm1 // save %rdi + movq %rcx,%xmm2 // save %rcx + movq %r10,%xmm3 // -%r9 movq %rdx,%xmm4 movq %r8,32(%rsp) - movq %rax,40(%rsp) + movq %rax,40(%rsp) // save original %rsp .cfi_escape 0x0f,0x05,0x77,0x28,0x06,0x23,0x08 .Lpowerx5_body: @@ -2709,7 +2709,7 @@ call __bn_sqrx8x_internal call __bn_postx4x_internal - movq %r10,%r9 + movq %r10,%r9 // -num movq %rsi,%rdi movq %xmm2,%rcx movq %xmm4,%rdx @@ -2717,7 +2717,7 @@ call mulx4x_internal - movq 40(%rsp),%rsi + movq 40(%rsp),%rsi // restore %rsp .cfi_def_cfa %rsi,8 movq $1,%rax @@ -2749,50 +2749,50 @@ __bn_sqrx8x_internal: .cfi_startproc _CET_ENDBR - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// ################################################################# +// Squaring part: +// +// a) multiply-n-add everything but a[i]*a[i]; +// b) shift result of a) by 1 to the left and accumulate +// a[i]*a[i] products; +// +// ################################################################# +// a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] +// a[1]a[0] +// a[2]a[0] +// a[3]a[0] +// a[2]a[1] +// a[3]a[1] +// a[3]a[2] +// +// a[4]a[0] +// a[5]a[0] +// a[6]a[0] +// a[7]a[0] +// a[4]a[1] +// a[5]a[1] +// a[6]a[1] +// a[7]a[1] +// a[4]a[2] +// a[5]a[2] +// a[6]a[2] +// a[7]a[2] +// a[4]a[3] +// a[5]a[3] +// a[6]a[3] +// a[7]a[3] +// +// a[5]a[4] +// a[6]a[4] +// a[7]a[4] +// a[6]a[5] +// a[7]a[5] +// a[7]a[6] +// a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] leaq 48+8(%rsp),%rdi leaq (%rsi,%r9,1),%rbp - movq %r9,0+8(%rsp) - movq %rbp,8+8(%rsp) + movq %r9,0+8(%rsp) // save %r9 + movq %rbp,8+8(%rsp) // save end of %rsi jmp .Lsqr8x_zero_start .align 32 @@ -2803,7 +2803,7 @@ movdqa %xmm0,16(%rdi) movdqa %xmm0,32(%rdi) movdqa %xmm0,48(%rdi) -.Lsqr8x_zero_start: +.Lsqr8x_zero_start: // aligned at 32 movdqa %xmm0,64(%rdi) movdqa %xmm0,80(%rdi) movdqa %xmm0,96(%rdi) @@ -2812,8 +2812,8 @@ subq $64,%r9 jnz .Lsqrx8x_zero - movq 0(%rsi),%rdx - + movq 0(%rsi),%rdx // a[0], modulo-scheduled +// xor %r9,%r9 # t[1], ex-%r9, zero already xorq %r10,%r10 xorq %r11,%r11 xorq %r12,%r12 @@ -2821,21 +2821,21 @@ xorq %r14,%r14 xorq %r15,%r15 leaq 48+8(%rsp),%rdi - xorq %rbp,%rbp + xorq %rbp,%rbp // cf=0, cf=0 jmp .Lsqrx8x_outer_loop .align 32 .Lsqrx8x_outer_loop: - mulxq 8(%rsi),%r8,%rax - adcxq %r9,%r8 + mulxq 8(%rsi),%r8,%rax // a[1]*a[0] + adcxq %r9,%r8 // a[1]*a[0]+=t[1] adoxq %rax,%r10 - mulxq 16(%rsi),%r9,%rax + mulxq 16(%rsi),%r9,%rax // a[2]*a[0] adcxq %r10,%r9 adoxq %rax,%r11 -.byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 // mulx 3*8(%rsi),%r10,%rax # ... adcxq %r11,%r10 adoxq %rax,%r12 -.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 // mulx 4*8(%rsi),%r11,%rax adcxq %r12,%r11 adoxq %rax,%r13 mulxq 40(%rsi),%r12,%rax @@ -2845,97 +2845,97 @@ adcxq %r14,%r13 adoxq %r15,%rax mulxq 56(%rsi),%r14,%r15 - movq 8(%rsi),%rdx + movq 8(%rsi),%rdx // a[1] adcxq %rax,%r14 adoxq %rbp,%r15 adcq 64(%rdi),%r15 - movq %r8,8(%rdi) - movq %r9,16(%rdi) - sbbq %rcx,%rcx - xorq %rbp,%rbp + movq %r8,8(%rdi) // t[1] + movq %r9,16(%rdi) // t[2] + sbbq %rcx,%rcx // mov %cf,%rcx + xorq %rbp,%rbp // cf=0, of=0 - mulxq 16(%rsi),%r8,%rbx - mulxq 24(%rsi),%r9,%rax + mulxq 16(%rsi),%r8,%rbx // a[2]*a[1] + mulxq 24(%rsi),%r9,%rax // a[3]*a[1] adcxq %r10,%r8 adoxq %rbx,%r9 - mulxq 32(%rsi),%r10,%rbx + mulxq 32(%rsi),%r10,%rbx // ... adcxq %r11,%r9 adoxq %rax,%r10 -.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 // mulx 5*8(%rsi),%r11,%rax adcxq %r12,%r10 adoxq %rbx,%r11 -.byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 +.byte 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 // mulx 6*8(%rsi),%r12,%rbx adcxq %r13,%r11 adoxq %r14,%r12 -.byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 - movq 16(%rsi),%rdx +.byte 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 // mulx 7*8(%rsi),%r13,%r14 + movq 16(%rsi),%rdx // a[2] adcxq %rax,%r12 adoxq %rbx,%r13 adcxq %r15,%r13 - adoxq %rbp,%r14 - adcxq %rbp,%r14 + adoxq %rbp,%r14 // of=0 + adcxq %rbp,%r14 // cf=0 - movq %r8,24(%rdi) - movq %r9,32(%rdi) + movq %r8,24(%rdi) // t[3] + movq %r9,32(%rdi) // t[4] - mulxq 24(%rsi),%r8,%rbx - mulxq 32(%rsi),%r9,%rax + mulxq 24(%rsi),%r8,%rbx // a[3]*a[2] + mulxq 32(%rsi),%r9,%rax // a[4]*a[2] adcxq %r10,%r8 adoxq %rbx,%r9 - mulxq 40(%rsi),%r10,%rbx + mulxq 40(%rsi),%r10,%rbx // ... adcxq %r11,%r9 adoxq %rax,%r10 -.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 +.byte 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 // mulx 6*8(%rsi),%r11,%rax adcxq %r12,%r10 adoxq %r13,%r11 -.byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 +.byte 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 // mulx 7*8(%rsi),%r12,%r13 .byte 0x3e - movq 24(%rsi),%rdx + movq 24(%rsi),%rdx // a[3] adcxq %rbx,%r11 adoxq %rax,%r12 adcxq %r14,%r12 - movq %r8,40(%rdi) - movq %r9,48(%rdi) - mulxq 32(%rsi),%r8,%rax - adoxq %rbp,%r13 - adcxq %rbp,%r13 + movq %r8,40(%rdi) // t[5] + movq %r9,48(%rdi) // t[6] + mulxq 32(%rsi),%r8,%rax // a[4]*a[3] + adoxq %rbp,%r13 // of=0 + adcxq %rbp,%r13 // cf=0 - mulxq 40(%rsi),%r9,%rbx + mulxq 40(%rsi),%r9,%rbx // a[5]*a[3] adcxq %r10,%r8 adoxq %rax,%r9 - mulxq 48(%rsi),%r10,%rax + mulxq 48(%rsi),%r10,%rax // ... adcxq %r11,%r9 adoxq %r12,%r10 mulxq 56(%rsi),%r11,%r12 - movq 32(%rsi),%rdx - movq 40(%rsi),%r14 + movq 32(%rsi),%rdx // a[4] + movq 40(%rsi),%r14 // a[5] adcxq %rbx,%r10 adoxq %rax,%r11 - movq 48(%rsi),%r15 + movq 48(%rsi),%r15 // a[6] adcxq %r13,%r11 - adoxq %rbp,%r12 - adcxq %rbp,%r12 + adoxq %rbp,%r12 // of=0 + adcxq %rbp,%r12 // cf=0 - movq %r8,56(%rdi) - movq %r9,64(%rdi) + movq %r8,56(%rdi) // t[7] + movq %r9,64(%rdi) // t[8] - mulxq %r14,%r9,%rax - movq 56(%rsi),%r8 + mulxq %r14,%r9,%rax // a[5]*a[4] + movq 56(%rsi),%r8 // a[7] adcxq %r10,%r9 - mulxq %r15,%r10,%rbx + mulxq %r15,%r10,%rbx // a[6]*a[4] adoxq %rax,%r10 adcxq %r11,%r10 - mulxq %r8,%r11,%rax - movq %r14,%rdx + mulxq %r8,%r11,%rax // a[7]*a[4] + movq %r14,%rdx // a[5] adoxq %rbx,%r11 adcxq %r12,%r11 +// adox %rbp,%rax # of=0 + adcxq %rbp,%rax // cf=0 - adcxq %rbp,%rax - - mulxq %r15,%r14,%rbx - mulxq %r8,%r12,%r13 - movq %r15,%rdx + mulxq %r15,%r14,%rbx // a[6]*a[5] + mulxq %r8,%r12,%r13 // a[7]*a[5] + movq %r15,%rdx // a[6] leaq 64(%rsi),%rsi adcxq %r14,%r11 adoxq %rbx,%r12 @@ -2943,19 +2943,19 @@ adoxq %rbp,%r13 .byte 0x67,0x67 - mulxq %r8,%r8,%r14 + mulxq %r8,%r8,%r14 // a[7]*a[6] adcxq %r8,%r13 adcxq %rbp,%r14 cmpq 8+8(%rsp),%rsi je .Lsqrx8x_outer_break - negq %rcx + negq %rcx // mov %rcx,%cf movq $-8,%rcx movq %rbp,%r15 movq 64(%rdi),%r8 - adcxq 72(%rdi),%r9 - adcxq 80(%rdi),%r10 + adcxq 72(%rdi),%r9 // +=t[9] + adcxq 80(%rdi),%r10 // ... adcxq 88(%rdi),%r11 adcq 96(%rdi),%r12 adcq 104(%rdi),%r13 @@ -2963,24 +2963,24 @@ adcq 120(%rdi),%r15 leaq (%rsi),%rbp leaq 128(%rdi),%rdi - sbbq %rax,%rax + sbbq %rax,%rax // mov %cf,%rcx - movq -64(%rsi),%rdx - movq %rax,16+8(%rsp) + movq -64(%rsi),%rdx // a[0] + movq %rax,16+8(%rsp) // offload %rcx movq %rdi,24+8(%rsp) - - xorl %eax,%eax +// lea 8*8(%rdi),%rdi # see 2*8*8(%rdi) above + xorl %eax,%eax // cf=0, of=0 jmp .Lsqrx8x_loop .align 32 .Lsqrx8x_loop: movq %r8,%rbx - mulxq 0(%rbp),%rax,%r8 - adcxq %rax,%rbx + mulxq 0(%rbp),%rax,%r8 // a[8]*a[i] + adcxq %rax,%rbx // +=t[8] adoxq %r9,%r8 - mulxq 8(%rbp),%rax,%r9 + mulxq 8(%rbp),%rax,%r9 // ... adcxq %rax,%r8 adoxq %r10,%r9 @@ -2992,7 +2992,7 @@ adcxq %rax,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 +.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 // mulx 4*8(%rbp),%rax,%r12 adcxq %rax,%r11 adoxq %r13,%r12 @@ -3001,27 +3001,27 @@ adoxq %r14,%r13 mulxq 48(%rbp),%rax,%r14 - movq %rbx,(%rdi,%rcx,8) + movq %rbx,(%rdi,%rcx,8) // store t[8+i] movl $0,%ebx adcxq %rax,%r13 adoxq %r15,%r14 -.byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 - movq 8(%rsi,%rcx,8),%rdx +.byte 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 // mulx 7*8(%rbp),%rax,%r15 + movq 8(%rsi,%rcx,8),%rdx // a[i] adcxq %rax,%r14 - adoxq %rbx,%r15 - adcxq %rbx,%r15 + adoxq %rbx,%r15 // %rbx is 0, of=0 + adcxq %rbx,%r15 // cf=0 .byte 0x67 - incq %rcx + incq %rcx // of=0 jnz .Lsqrx8x_loop leaq 64(%rbp),%rbp movq $-8,%rcx - cmpq 8+8(%rsp),%rbp + cmpq 8+8(%rsp),%rbp // done? je .Lsqrx8x_break - subq 16+8(%rsp),%rbx + subq 16+8(%rsp),%rbx // mov 16(%rsp),%cf .byte 0x66 movq -64(%rsi),%rdx adcxq 0(%rdi),%r8 @@ -3034,19 +3034,19 @@ adcq 56(%rdi),%r15 leaq 64(%rdi),%rdi .byte 0x67 - sbbq %rax,%rax - xorl %ebx,%ebx - movq %rax,16+8(%rsp) + sbbq %rax,%rax // mov %cf,%rax + xorl %ebx,%ebx // cf=0, of=0 + movq %rax,16+8(%rsp) // offload carry jmp .Lsqrx8x_loop .align 32 .Lsqrx8x_break: xorq %rbp,%rbp - subq 16+8(%rsp),%rbx + subq 16+8(%rsp),%rbx // mov 16(%rsp),%cf adcxq %rbp,%r8 - movq 24+8(%rsp),%rcx + movq 24+8(%rsp),%rcx // initial %rdi, borrow %rcx adcxq %rbp,%r9 - movq 0(%rsi),%rdx + movq 0(%rsi),%rdx // a[8], modulo-scheduled adcq $0,%r10 movq %r8,0(%rdi) adcq $0,%r11 @@ -3054,7 +3054,7 @@ adcq $0,%r13 adcq $0,%r14 adcq $0,%r15 - cmpq %rcx,%rdi + cmpq %rcx,%rdi // cf=0, of=0 je .Lsqrx8x_outer_loop movq %r9,8(%rdi) @@ -3076,57 +3076,57 @@ .align 32 .Lsqrx8x_outer_break: - movq %r9,72(%rdi) - movq %xmm3,%rcx - movq %r10,80(%rdi) + movq %r9,72(%rdi) // t[9] + movq %xmm3,%rcx // -%r9 + movq %r10,80(%rdi) // ... movq %r11,88(%rdi) movq %r12,96(%rdi) movq %r13,104(%rdi) movq %r14,112(%rdi) leaq 48+8(%rsp),%rdi - movq (%rsi,%rcx,1),%rdx + movq (%rsi,%rcx,1),%rdx // a[0] - movq 8(%rdi),%r11 - xorq %r10,%r10 - movq 0+8(%rsp),%r9 + movq 8(%rdi),%r11 // t[1] + xorq %r10,%r10 // t[0], of=0, cf=0 + movq 0+8(%rsp),%r9 // restore %r9 adoxq %r11,%r11 - movq 16(%rdi),%r12 - movq 24(%rdi),%r13 - + movq 16(%rdi),%r12 // t[2] # prefetch + movq 24(%rdi),%r13 // t[3] # prefetch +// jmp .Lsqrx4x_shift_n_add # happens to be aligned .align 32 .Lsqrx4x_shift_n_add: mulxq %rdx,%rax,%rbx adoxq %r12,%r12 adcxq %r10,%rax -.byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 -.byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 +.byte 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 // mov 8(%rsi,%rcx),%rdx # a[i+1] # prefetch +.byte 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 // mov 32(%rdi),%r10 # t[2*i+4] # prefetch adoxq %r13,%r13 adcxq %r11,%rbx - movq 40(%rdi),%r11 + movq 40(%rdi),%r11 // t[2*i+4+1] # prefetch movq %rax,0(%rdi) movq %rbx,8(%rdi) mulxq %rdx,%rax,%rbx adoxq %r10,%r10 adcxq %r12,%rax - movq 16(%rsi,%rcx,1),%rdx - movq 48(%rdi),%r12 + movq 16(%rsi,%rcx,1),%rdx // a[i+2] # prefetch + movq 48(%rdi),%r12 // t[2*i+6] # prefetch adoxq %r11,%r11 adcxq %r13,%rbx - movq 56(%rdi),%r13 + movq 56(%rdi),%r13 // t[2*i+6+1] # prefetch movq %rax,16(%rdi) movq %rbx,24(%rdi) mulxq %rdx,%rax,%rbx adoxq %r12,%r12 adcxq %r10,%rax - movq 24(%rsi,%rcx,1),%rdx + movq 24(%rsi,%rcx,1),%rdx // a[i+3] # prefetch leaq 32(%rcx),%rcx - movq 64(%rdi),%r10 + movq 64(%rdi),%r10 // t[2*i+8] # prefetch adoxq %r13,%r13 adcxq %r11,%rbx - movq 72(%rdi),%r11 + movq 72(%rdi),%r11 // t[2*i+8+1] # prefetch movq %rax,32(%rdi) movq %rbx,40(%rdi) @@ -3134,11 +3134,11 @@ adoxq %r10,%r10 adcxq %r12,%rax jrcxz .Lsqrx4x_shift_n_add_break -.byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 +.byte 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 // mov 0(%rsi,%rcx),%rdx # a[i+4] # prefetch adoxq %r11,%r11 adcxq %r13,%rbx - movq 80(%rdi),%r12 - movq 88(%rdi),%r13 + movq 80(%rdi),%r12 // t[2*i+10] # prefetch + movq 88(%rdi),%r13 // t[2*i+10+1] # prefetch movq %rax,48(%rdi) movq %rbx,56(%rdi) leaq 64(%rdi),%rdi @@ -3150,18 +3150,18 @@ adcxq %r13,%rbx movq %rax,48(%rdi) movq %rbx,56(%rdi) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // end of t[] buffer movq %xmm2,%rbp __bn_sqrx8x_reduction: - xorl %eax,%eax - movq 32+8(%rsp),%rbx - movq 48+8(%rsp),%rdx - leaq -64(%rbp,%r9,1),%rcx + xorl %eax,%eax // initial top-most carry bit + movq 32+8(%rsp),%rbx // n0 + movq 48+8(%rsp),%rdx // "%r8", 8*0(%rdi) + leaq -64(%rbp,%r9,1),%rcx // end of n[] +// lea 48+8(%rsp,%r9,2),%rdi # end of t[] buffer + movq %rcx,0+8(%rsp) // save end of n[] + movq %rdi,8+8(%rsp) // save end of t[] - movq %rcx,0+8(%rsp) - movq %rdi,8+8(%rsp) - - leaq 48+8(%rsp),%rdi + leaq 48+8(%rsp),%rdi // initial t[] window jmp .Lsqrx8x_reduction_loop .align 32 @@ -3171,25 +3171,25 @@ movq 24(%rdi),%r11 movq 32(%rdi),%r12 movq %rdx,%r8 - imulq %rbx,%rdx + imulq %rbx,%rdx // n0*a[i] movq 40(%rdi),%r13 movq 48(%rdi),%r14 movq 56(%rdi),%r15 - movq %rax,24+8(%rsp) + movq %rax,24+8(%rsp) // store top-most carry bit leaq 64(%rdi),%rdi - xorq %rsi,%rsi + xorq %rsi,%rsi // cf=0,of=0 movq $-8,%rcx jmp .Lsqrx8x_reduce .align 32 .Lsqrx8x_reduce: movq %r8,%rbx - mulxq 0(%rbp),%rax,%r8 - adcxq %rbx,%rax + mulxq 0(%rbp),%rax,%r8 // n[0] + adcxq %rbx,%rax // discarded adoxq %r9,%r8 - mulxq 8(%rbp),%rbx,%r9 + mulxq 8(%rbp),%rbx,%r9 // n[1] adcxq %rbx,%r8 adoxq %r10,%r9 @@ -3201,15 +3201,15 @@ adcxq %rbx,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 +.byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 // mulx 8*4(%rbp),%rbx,%r12 movq %rdx,%rax movq %r8,%rdx adcxq %rbx,%r11 adoxq %r13,%r12 - mulxq 32+8(%rsp),%rbx,%rdx + mulxq 32+8(%rsp),%rbx,%rdx // %rdx discarded movq %rax,%rdx - movq %rax,64+48+8(%rsp,%rcx,8) + movq %rax,64+48+8(%rsp,%rcx,8) // put aside n0*a[i] mulxq 40(%rbp),%rax,%r13 adcxq %rax,%r12 @@ -3222,18 +3222,18 @@ mulxq 56(%rbp),%rax,%r15 movq %rbx,%rdx adcxq %rax,%r14 - adoxq %rsi,%r15 - adcxq %rsi,%r15 + adoxq %rsi,%r15 // %rsi is 0 + adcxq %rsi,%r15 // cf=0 .byte 0x67,0x67,0x67 - incq %rcx + incq %rcx // of=0 jnz .Lsqrx8x_reduce - movq %rsi,%rax - cmpq 0+8(%rsp),%rbp + movq %rsi,%rax // xor %rax,%rax + cmpq 0+8(%rsp),%rbp // end of n[]? jae .Lsqrx8x_no_tail - movq 48+8(%rsp),%rdx + movq 48+8(%rsp),%rdx // pull n0*a[0] addq 0(%rdi),%r8 leaq 64(%rbp),%rbp movq $-8,%rcx @@ -3245,9 +3245,9 @@ adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 leaq 64(%rdi),%rdi - sbbq %rax,%rax + sbbq %rax,%rax // top carry - xorq %rsi,%rsi + xorq %rsi,%rsi // of=0, cf=0 movq %rax,16+8(%rsp) jmp .Lsqrx8x_tail @@ -3270,7 +3270,7 @@ adcxq %rax,%r10 adoxq %r12,%r11 -.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 +.byte 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 // mulx 8*4(%rbp),%rax,%r12 adcxq %rax,%r11 adoxq %r13,%r12 @@ -3283,21 +3283,21 @@ adoxq %r15,%r14 mulxq 56(%rbp),%rax,%r15 - movq 72+48+8(%rsp,%rcx,8),%rdx + movq 72+48+8(%rsp,%rcx,8),%rdx // pull n0*a[i] adcxq %rax,%r14 adoxq %rsi,%r15 - movq %rbx,(%rdi,%rcx,8) + movq %rbx,(%rdi,%rcx,8) // save result movq %r8,%rbx - adcxq %rsi,%r15 + adcxq %rsi,%r15 // cf=0 - incq %rcx + incq %rcx // of=0 jnz .Lsqrx8x_tail - cmpq 0+8(%rsp),%rbp - jae .Lsqrx8x_tail_done + cmpq 0+8(%rsp),%rbp // end of n[]? + jae .Lsqrx8x_tail_done // break out of loop - subq 16+8(%rsp),%rsi - movq 48+8(%rsp),%rdx + subq 16+8(%rsp),%rsi // mov 16(%rsp),%cf + movq 48+8(%rsp),%rdx // pull n0*a[0] leaq 64(%rbp),%rbp adcq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -3309,16 +3309,16 @@ adcq 56(%rdi),%r15 leaq 64(%rdi),%rdi sbbq %rax,%rax - subq $8,%rcx + subq $8,%rcx // mov $-8,%rcx - xorq %rsi,%rsi + xorq %rsi,%rsi // of=0, cf=0 movq %rax,16+8(%rsp) jmp .Lsqrx8x_tail .align 32 .Lsqrx8x_tail_done: xorq %rax,%rax - addq 24+8(%rsp),%r8 + addq 24+8(%rsp),%r8 // can this overflow? adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 @@ -3328,26 +3328,26 @@ adcq $0,%r15 adcq $0,%rax - subq 16+8(%rsp),%rsi -.Lsqrx8x_no_tail: + subq 16+8(%rsp),%rsi // mov 16(%rsp),%cf +.Lsqrx8x_no_tail: // %cf is 0 if jumped here adcq 0(%rdi),%r8 movq %xmm3,%rcx adcq 8(%rdi),%r9 movq 56(%rbp),%rsi - movq %xmm2,%rbp + movq %xmm2,%rbp // restore %rbp adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 adcq 48(%rdi),%r14 adcq 56(%rdi),%r15 - adcq $0,%rax + adcq $0,%rax // top-most carry - movq 32+8(%rsp),%rbx - movq 64(%rdi,%rcx,1),%rdx + movq 32+8(%rsp),%rbx // n0 + movq 64(%rdi,%rcx,1),%rdx // modulo-scheduled "%r8" - movq %r8,0(%rdi) - leaq 64(%rdi),%r8 + movq %r8,0(%rdi) // store top 512 bits + leaq 64(%rdi),%r8 // borrow %r8 movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) @@ -3356,8 +3356,8 @@ movq %r14,48(%rdi) movq %r15,56(%rdi) - leaq 64(%rdi,%rcx,1),%rdi - cmpq 8+8(%rsp),%r8 + leaq 64(%rdi,%rcx,1),%rdi // start of current t[] window + cmpq 8+8(%rsp),%r8 // end of t[]? jb .Lsqrx8x_reduction_loop ret .cfi_endproc @@ -3367,14 +3367,14 @@ __bn_postx4x_internal: .cfi_startproc movq 0(%rbp),%r12 - movq %rcx,%r10 - movq %rcx,%r9 + movq %rcx,%r10 // -%r9 + movq %rcx,%r9 // -%r9 negq %rax sarq $3+2,%rcx - - movq %xmm1,%rdx - movq %xmm1,%rsi - decq %r12 +// lea 48+8(%rsp,%r9),%rdi + movq %xmm1,%rdx // restore %rdx + movq %xmm1,%rsi // prepare for back-to-back call + decq %r12 // so that after 'not' we get -n[0] movq 8(%rbp),%r13 xorq %r8,%r8 movq 16(%rbp),%r14 @@ -3394,7 +3394,7 @@ andnq %rax,%r14,%r14 andnq %rax,%r15,%r15 - negq %r8 + negq %r8 // mov %r8,%cf adcq 0(%rdi),%r12 adcq 8(%rdi),%r13 adcq 16(%rdi),%r14 @@ -3402,7 +3402,7 @@ movq %r12,0(%rdx) leaq 32(%rdi),%rdi movq %r13,8(%rdx) - sbbq %r8,%r8 + sbbq %r8,%r8 // mov %cf,%r8 movq %r14,16(%rdx) movq %r15,24(%rdx) leaq 32(%rdx),%rdx @@ -3410,7 +3410,7 @@ incq %rcx jnz .Lsqrx4x_sub - negq %r9 + negq %r9 // restore %r9 ret .cfi_endproc @@ -3425,13 +3425,13 @@ cmpl $0,%esi jz .Lscatter_epilogue - - - - - - - +// %rdx stores 32 entries, t0 through t31. Each entry has %esi words. +// They are interleaved in memory as follows: +// +// t0[0] t1[0] t2[0] ... t31[0] +// t0[1] t1[1] t2[1] ... t31[1] +// ... +// t0[%esi-1] t1[%esi-1] t2[%esi-1] ... t31[%esi-1] leaq (%rdx,%rcx,8),%rdx .Lscatter: @@ -3452,97 +3452,97 @@ .align 32 bn_gather5: .cfi_startproc -.LSEH_begin_bn_gather5: +.LSEH_begin_bn_gather5: // Win64 thing, but harmless in other cases _CET_ENDBR - -.byte 0x4c,0x8d,0x14,0x24 +// I can't trust assembler to use specific encoding:-( +.byte 0x4c,0x8d,0x14,0x24 // lea (%rsp),%r10 .cfi_def_cfa_register %r10 -.byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 +.byte 0x48,0x81,0xec,0x08,0x01,0x00,0x00 // sub $0x108,%rsp leaq .Linc(%rip),%rax - andq $-16,%rsp + andq $-16,%rsp // shouldn't be formally required movd %ecx,%xmm5 - movdqa 0(%rax),%xmm0 - movdqa 16(%rax),%xmm1 - leaq 128(%rdx),%r11 - leaq 128(%rsp),%rax + movdqa 0(%rax),%xmm0 // 00000001000000010000000000000000 + movdqa 16(%rax),%xmm1 // 00000002000000020000000200000002 + leaq 128(%rdx),%r11 // size optimization + leaq 128(%rsp),%rax // size optimization - pshufd $0,%xmm5,%xmm5 + pshufd $0,%xmm5,%xmm5 // broadcast %ecx movdqa %xmm1,%xmm4 movdqa %xmm1,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,-128(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,-112(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,-96(%rax) movdqa %xmm4,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm3,-80(%rax) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,-64(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,-48(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,-32(%rax) movdqa %xmm4,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm3,-16(%rax) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,0(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,16(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,32(%rax) movdqa %xmm4,%xmm2 paddd %xmm0,%xmm1 - pcmpeqd %xmm5,%xmm0 + pcmpeqd %xmm5,%xmm0 // compare to 1,0 movdqa %xmm3,48(%rax) movdqa %xmm4,%xmm3 paddd %xmm1,%xmm2 - pcmpeqd %xmm5,%xmm1 + pcmpeqd %xmm5,%xmm1 // compare to 3,2 movdqa %xmm0,64(%rax) movdqa %xmm4,%xmm0 paddd %xmm2,%xmm3 - pcmpeqd %xmm5,%xmm2 + pcmpeqd %xmm5,%xmm2 // compare to 5,4 movdqa %xmm1,80(%rax) movdqa %xmm4,%xmm1 paddd %xmm3,%xmm0 - pcmpeqd %xmm5,%xmm3 + pcmpeqd %xmm5,%xmm3 // compare to 7,6 movdqa %xmm2,96(%rax) movdqa %xmm4,%xmm2 movdqa %xmm3,112(%rax) @@ -3602,10 +3602,10 @@ por %xmm3,%xmm5 por %xmm5,%xmm4 leaq 256(%r11),%r11 - - pshufd $0x4e,%xmm4,%xmm0 +// Combine the upper and lower halves of %xmm0. + pshufd $0x4e,%xmm4,%xmm0 // Swap upper and lower halves. por %xmm4,%xmm0 - movq %xmm0,(%rdi) + movq %xmm0,(%rdi) // m0=bp[0] leaq 8(%rdi),%rdi subl $1,%esi jnz .Lgather
diff --git a/gen/bcm/x86_64-mont5-win.asm b/gen/bcm/x86_64-mont5-win.asm index 1afb44f..05e9759 100644 --- a/gen/bcm/x86_64-mont5-win.asm +++ b/gen/bcm/x86_64-mont5-win.asm
@@ -32,12 +32,12 @@ _CET_ENDBR - - +; num is declared as an int, a 32-bit parameter, so the upper half is +; undefined. Zero the upper half to normalize it. mov r9d,r9d mov rax,rsp - movd xmm5,DWORD[56+rsp] + movd xmm5,DWORD[56+rsp] ; load 7th argument push rbx push rbp @@ -53,18 +53,18 @@ neg r9 mov r11,rsp - lea r10,[((-280))+r9*8+rsp] - neg r9 - and r10,-1024 + lea r10,[((-280))+r9*8+rsp] ; future alloca(8*(num+2)+256+8) + neg r9 ; restore %r9 + and r10,-1024 ; minimize TLB usage - - - - - - - - +; An OS-agnostic version of __chkstk. +; +; Some OSes (Windows) insist on stack being "wired" to +; physical memory in strictly sequential manner, i.e. if stack +; allocation spans two pages, then reference to farmost one can +; be punishable by SEGV. But page walking can do good even on +; other OSes, because it guarantees that villain thread hits +; the guard page before it can make damage to innocent one... sub r11,r10 and r11,-4096 lea rsp,[r11*1+r10] @@ -81,35 +81,35 @@ $L$mul_page_walk_done: lea r10,[$L$inc] - mov QWORD[8+r9*8+rsp],rax + mov QWORD[8+r9*8+rsp],rax ; tp[num+1]=%rsp $L$mul_body: - lea r12,[128+rdx] - movdqa xmm0,XMMWORD[r10] - movdqa xmm1,XMMWORD[16+r10] - lea r10,[((24-112))+r9*8+rsp] + lea r12,[128+rdx] ; reassign %rdx (+size optimization) + movdqa xmm0,XMMWORD[r10] ; 00000001000000010000000000000000 + movdqa xmm1,XMMWORD[16+r10] ; 00000002000000020000000200000002 + lea r10,[((24-112))+r9*8+rsp] ; place the mask after tp[num+3] (+ICache optimization) and r10,-16 - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; broadcast index movdqa xmm4,xmm1 movdqa xmm2,xmm1 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 DB 0x67 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[112+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[128+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[144+r10],xmm2 movdqa xmm2,xmm4 @@ -118,17 +118,17 @@ movdqa XMMWORD[160+r10],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[176+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[192+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[208+r10],xmm2 movdqa xmm2,xmm4 @@ -137,17 +137,17 @@ movdqa XMMWORD[224+r10],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[240+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[256+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[272+r10],xmm2 movdqa xmm2,xmm4 @@ -166,7 +166,7 @@ pcmpeqd xmm3,xmm5 movdqa XMMWORD[336+r10],xmm2 - pand xmm0,XMMWORD[64+r12] + pand xmm0,XMMWORD[64+r12] ; while it's still in register pand xmm1,XMMWORD[80+r12] pand xmm2,XMMWORD[96+r12] @@ -211,33 +211,33 @@ por xmm0,xmm2 por xmm1,xmm3 por xmm0,xmm1 - - pshufd xmm1,xmm0,0x4e +; Combine the upper and lower halves of %xmm0. + pshufd xmm1,xmm0,0x4e ; Swap upper and lower halves. por xmm0,xmm1 lea r12,[256+r12] - movq rbx,xmm0 + movq rbx,xmm0 ; m0=bp[0] - mov r8,QWORD[r8] + mov r8,QWORD[r8] ; pull n0[0] value mov rax,QWORD[rsi] - xor r14,r14 - xor r15,r15 + xor r14,r14 ; i=0 + xor r15,r15 ; j=0 mov rbp,r8 - mul rbx + mul rbx ; ap[0]*bp[0] mov r10,rax mov rax,QWORD[rcx] - imul rbp,r10 + imul rbp,r10 ; "tp[0]"*n0 mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; discarded mov rax,QWORD[8+rsi] adc rdx,0 mov r13,rdx - lea r15,[1+r15] + lea r15,[1+r15] ; j++ jmp NEAR $L$1st_enter ALIGN 16 @@ -245,30 +245,30 @@ add r13,rax mov rax,QWORD[r15*8+rsi] adc rdx,0 - add r13,r11 + add r13,r11 ; np[j]*m1+ap[j]*bp[0] mov r11,r10 adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],r13 + mov QWORD[((-16))+r15*8+rsp],r13 ; tp[j-1] mov r13,rdx $L$1st_enter: - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[r15*8+rcx] adc rdx,0 - lea r15,[1+r15] + lea r15,[1+r15] ; j++ mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 cmp r15,r9 - jne NEAR $L$1st - + jne NEAR $L$1st ; note that upon exit %r15==%r9, so +; they can be used interchangeably add r13,rax adc rdx,0 - add r13,r11 + add r13,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-16))+r9*8+rsp],r13 + mov QWORD[((-16))+r9*8+rsp],r13 ; tp[num-1] mov r13,rdx mov r11,r10 @@ -276,13 +276,13 @@ add r13,r11 adc rdx,0 mov QWORD[((-8))+r9*8+rsp],r13 - mov QWORD[r9*8+rsp],rdx + mov QWORD[r9*8+rsp],rdx ; store upmost overflow bit - lea r14,[1+r14] + lea r14,[1+r14] ; i++ jmp NEAR $L$outer ALIGN 16 $L$outer: - lea rdx,[((24+128))+r9*8+rsp] + lea rdx,[((24+128))+r9*8+rsp] ; where 256-byte mask is (+size optimization) and rdx,-16 pxor xmm4,xmm4 pxor xmm5,xmm5 @@ -335,34 +335,34 @@ por xmm4,xmm2 por xmm5,xmm3 por xmm4,xmm5 - - pshufd xmm0,xmm4,0x4e +; Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd xmm0,xmm4,0x4e ; Swap upper and lower halves. por xmm0,xmm4 lea r12,[256+r12] - mov rax,QWORD[rsi] - movq rbx,xmm0 + mov rax,QWORD[rsi] ; ap[0] + movq rbx,xmm0 ; m0=bp[i] - xor r15,r15 + xor r15,r15 ; j=0 mov rbp,r8 mov r10,QWORD[rsp] - mul rbx - add r10,rax + mul rbx ; ap[0]*bp[i] + add r10,rax ; ap[0]*bp[i]+tp[0] mov rax,QWORD[rcx] adc rdx,0 - imul rbp,r10 + imul rbp,r10 ; tp[0]*n0 mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; discarded mov rax,QWORD[8+rsi] adc rdx,0 - mov r10,QWORD[8+rsp] + mov r10,QWORD[8+rsp] ; tp[1] mov r13,rdx - lea r15,[1+r15] + lea r15,[1+r15] ; j++ jmp NEAR $L$inner_enter ALIGN 16 @@ -370,78 +370,78 @@ add r13,rax mov rax,QWORD[r15*8+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[i]+tp[j] mov r10,QWORD[r15*8+rsp] adc rdx,0 - mov QWORD[((-16))+r15*8+rsp],r13 + mov QWORD[((-16))+r15*8+rsp],r13 ; tp[j-1] mov r13,rdx $L$inner_enter: - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[r15*8+rcx] adc rdx,0 - add r10,r11 + add r10,r11 ; ap[j]*bp[i]+tp[j] mov r11,rdx adc r11,0 - lea r15,[1+r15] + lea r15,[1+r15] ; j++ - mul rbp + mul rbp ; np[j]*m1 cmp r15,r9 - jne NEAR $L$inner - + jne NEAR $L$inner ; note that upon exit %r15==%r9, so +; they can be used interchangeably add r13,rax adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[i]+tp[j] mov r10,QWORD[r9*8+rsp] adc rdx,0 - mov QWORD[((-16))+r9*8+rsp],r13 + mov QWORD[((-16))+r9*8+rsp],r13 ; tp[num-1] mov r13,rdx xor rdx,rdx add r13,r11 adc rdx,0 - add r13,r10 + add r13,r10 ; pull upmost overflow bit adc rdx,0 mov QWORD[((-8))+r9*8+rsp],r13 - mov QWORD[r9*8+rsp],rdx + mov QWORD[r9*8+rsp],rdx ; store upmost overflow bit - lea r14,[1+r14] + lea r14,[1+r14] ; i++ cmp r14,r9 jb NEAR $L$outer - xor r14,r14 - mov rax,QWORD[rsp] - lea rsi,[rsp] - mov r15,r9 + xor r14,r14 ; i=0 and clear CF! + mov rax,QWORD[rsp] ; tp[0] + lea rsi,[rsp] ; borrow ap for tp + mov r15,r9 ; j=num jmp NEAR $L$sub ALIGN 16 $L$sub: sbb rax,QWORD[r14*8+rcx] - mov QWORD[r14*8+rdi],rax - mov rax,QWORD[8+r14*8+rsi] - lea r14,[1+r14] - dec r15 + mov QWORD[r14*8+rdi],rax ; rp[i]=tp[i]-np[i] + mov rax,QWORD[8+r14*8+rsi] ; tp[i+1] + lea r14,[1+r14] ; i++ + dec r15 ; doesn't affect CF! jnz NEAR $L$sub - sbb rax,0 + sbb rax,0 ; handle upmost overflow bit mov rbx,-1 xor rbx,rax xor r14,r14 - mov r15,r9 + mov r15,r9 ; j=num -$L$copy: +$L$copy: ; conditional copy mov rcx,QWORD[r14*8+rdi] mov rdx,QWORD[r14*8+rsp] and rcx,rbx and rdx,rax - mov QWORD[r14*8+rsp],r14 + mov QWORD[r14*8+rsp],r14 ; zap temporary vector or rdx,rcx - mov QWORD[r14*8+rdi],rdx + mov QWORD[r14*8+rdi],rdx ; rp[i]=tp[i] lea r14,[1+r14] sub r15,1 jnz NEAR $L$copy - mov rsi,QWORD[8+r9*8+rsp] + mov rsi,QWORD[8+r9*8+rsp] ; restore %rsp mov rax,1 @@ -501,36 +501,36 @@ $L$mul4x_prologue: DB 0x67 +; num is declared as an int, a 32-bit parameter, so the upper half is +; undefined. It is important that this write to %r9, which zeros the +; upper half, predates the first access. + shl r9d,3 ; convert %r9 to bytes + lea r10,[r9*2+r9] ; 3*%r9 in bytes + neg r9 ; -%r9 - - - shl r9d,3 - lea r10,[r9*2+r9] - neg r9 - - - - - - - - - - +; ############################################################# +; Ensure that stack frame doesn't alias with +3*%r9 +; modulo 4096, which covers ret[num], am[num] and n[num] +; (see bn_exp.c). This is done to allow memory disambiguation +; logic do its magic. [Extra [num] is allocated in order +; to align with bn_power5's frame, which is cleansed after +; completing exponentiation. Extra 256 bytes is for power mask +; calculated from 7th argument, the index.] +; lea r11,[((-320))+r9*2+rsp] mov rbp,rsp sub r11,rdi and r11,4095 cmp r10,r11 jb NEAR $L$mul4xsp_alt - sub rbp,r11 - lea rbp,[((-320))+r9*2+rbp] + sub rbp,r11 ; align with %rdi + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*num*8+256) jmp NEAR $L$mul4xsp_done ALIGN 32 $L$mul4xsp_alt: lea r10,[((4096-320))+r9*2] - lea rbp,[((-320))+r9*2+rbp] + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*num*8+256) sub r11,r10 mov r10,0 cmovc r11,r10 @@ -561,7 +561,7 @@ call mul4x_internal - mov rsi,QWORD[40+rsp] + mov rsi,QWORD[40+rsp] ; restore %rsp mov rax,1 @@ -590,36 +590,36 @@ ALIGN 32 mul4x_internal: - shl r9,5 - movd xmm5,DWORD[56+rax] + shl r9,5 ; %r9 was in bytes + movd xmm5,DWORD[56+rax] ; load 7th argument, index lea rax,[$L$inc] - lea r13,[128+r9*1+rdx] - shr r9,5 - movdqa xmm0,XMMWORD[rax] - movdqa xmm1,XMMWORD[16+rax] - lea r10,[((88-112))+r9*1+rsp] - lea r12,[128+rdx] + lea r13,[128+r9*1+rdx] ; end of powers table (+size optimization) + shr r9,5 ; restore %r9 + movdqa xmm0,XMMWORD[rax] ; 00000001000000010000000000000000 + movdqa xmm1,XMMWORD[16+rax] ; 00000002000000020000000200000002 + lea r10,[((88-112))+r9*1+rsp] ; place the mask after tp[num+1] (+ICache optimization) + lea r12,[128+rdx] ; size optimization - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; broadcast index movdqa xmm4,xmm1 DB 0x67,0x67 movdqa xmm2,xmm1 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 DB 0x67 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[112+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[128+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[144+r10],xmm2 movdqa xmm2,xmm4 @@ -628,17 +628,17 @@ movdqa XMMWORD[160+r10],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[176+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[192+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[208+r10],xmm2 movdqa xmm2,xmm4 @@ -647,17 +647,17 @@ movdqa XMMWORD[224+r10],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[240+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[256+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[272+r10],xmm2 movdqa xmm2,xmm4 @@ -676,7 +676,7 @@ pcmpeqd xmm3,xmm5 movdqa XMMWORD[336+r10],xmm2 - pand xmm0,XMMWORD[64+r12] + pand xmm0,XMMWORD[64+r12] ; while it's still in register pand xmm1,XMMWORD[80+r12] pand xmm2,XMMWORD[96+r12] @@ -721,31 +721,31 @@ por xmm0,xmm2 por xmm1,xmm3 por xmm0,xmm1 - - pshufd xmm1,xmm0,0x4e +; Combine the upper and lower halves of %xmm0. + pshufd xmm1,xmm0,0x4e ; Swap upper and lower halves. por xmm0,xmm1 lea r12,[256+r12] - movq rbx,xmm0 + movq rbx,xmm0 ; m0=bp[0] - mov QWORD[((16+8))+rsp],r13 - mov QWORD[((56+8))+rsp],rdi + mov QWORD[((16+8))+rsp],r13 ; save end of b[num] + mov QWORD[((56+8))+rsp],rdi ; save %rdi - mov r8,QWORD[r8] + mov r8,QWORD[r8] ; pull n0[0] value mov rax,QWORD[rsi] - lea rsi,[r9*1+rsi] + lea rsi,[r9*1+rsi] ; end of a[num] neg r9 mov rbp,r8 - mul rbx + mul rbx ; ap[0]*bp[0] mov r10,rax mov rax,QWORD[rcx] - imul rbp,r10 + imul rbp,r10 ; "tp[0]"*n0 lea r14,[((64+8))+rsp] mov r11,rdx - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; discarded mov rax,QWORD[8+r9*1+rsi] adc rdx,0 mov rdi,rdx @@ -761,7 +761,7 @@ mov rax,QWORD[16+r9*1+rsi] adc rdx,0 add rdi,r11 - lea r15,[32+r9] + lea r15,[32+r9] ; j=4 lea rcx,[32+rcx] adc rdx,0 mov QWORD[r14],rdi @@ -770,103 +770,103 @@ ALIGN 32 $L$1st4x: - mul rbx + mul rbx ; ap[j]*bp[0] add r10,rax mov rax,QWORD[((-16))+rcx] lea r14,[32+r14] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+r15*1+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-24))+r14],r13 + mov QWORD[((-24))+r14],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[((-8))+rcx] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[r15*1+rsi] adc rdx,0 - add rdi,r11 + add rdi,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-16))+r14],rdi + mov QWORD[((-16))+r14],rdi ; tp[j-1] mov r13,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r10,rax mov rax,QWORD[rcx] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[8+r15*1+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-8))+r14],r13 + mov QWORD[((-8))+r14],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[8+rcx] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[16+r15*1+rsi] adc rdx,0 - add rdi,r11 + add rdi,r11 ; np[j]*m1+ap[j]*bp[0] lea rcx,[32+rcx] adc rdx,0 - mov QWORD[r14],rdi + mov QWORD[r14],rdi ; tp[j-1] mov r13,rdx - add r15,32 + add r15,32 ; j+=4 jnz NEAR $L$1st4x - mul rbx + mul rbx ; ap[j]*bp[0] add r10,rax mov rax,QWORD[((-16))+rcx] lea r14,[32+r14] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+rsi] adc rdx,0 - add r13,r10 + add r13,r10 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-24))+r14],r13 + mov QWORD[((-24))+r14],r13 ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[0] add r11,rax mov rax,QWORD[((-8))+rcx] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax - mov rax,QWORD[r9*1+rsi] + mov rax,QWORD[r9*1+rsi] ; ap[0] adc rdx,0 - add rdi,r11 + add rdi,r11 ; np[j]*m1+ap[j]*bp[0] adc rdx,0 - mov QWORD[((-16))+r14],rdi + mov QWORD[((-16))+r14],rdi ; tp[j-1] mov r13,rdx - lea rcx,[r9*1+rcx] + lea rcx,[r9*1+rcx] ; rewind %rcx xor rdi,rdi add r13,r10 @@ -877,7 +877,7 @@ ALIGN 32 $L$outer4x: - lea rdx,[((16+128))+r14] + lea rdx,[((16+128))+r14] ; where 256-byte mask is (+size optimization) pxor xmm4,xmm4 pxor xmm5,xmm5 movdqa xmm0,XMMWORD[((-128))+r12] @@ -929,45 +929,45 @@ por xmm4,xmm2 por xmm5,xmm3 por xmm4,xmm5 - - pshufd xmm0,xmm4,0x4e +; Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd xmm0,xmm4,0x4e ; Swap upper and lower halves. por xmm0,xmm4 lea r12,[256+r12] - movq rbx,xmm0 + movq rbx,xmm0 ; m0=bp[i] mov r10,QWORD[r9*1+r14] mov rbp,r8 - mul rbx - add r10,rax + mul rbx ; ap[0]*bp[i] + add r10,rax ; ap[0]*bp[i]+tp[0] mov rax,QWORD[rcx] adc rdx,0 - imul rbp,r10 + imul rbp,r10 ; tp[0]*n0 mov r11,rdx - mov QWORD[r14],rdi + mov QWORD[r14],rdi ; store upmost overflow bit - lea r14,[r9*1+r14] + lea r14,[r9*1+r14] ; rewind %r14 - mul rbp - add r10,rax + mul rbp ; np[0]*m1 + add r10,rax ; "%r13", discarded mov rax,QWORD[8+r9*1+rsi] adc rdx,0 mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[8+rcx] adc rdx,0 - add r11,QWORD[8+r14] + add r11,QWORD[8+r14] ; +tp[1] adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[16+r9*1+rsi] adc rdx,0 - add rdi,r11 - lea r15,[32+r9] + add rdi,r11 ; np[j]*m1+ap[j]*bp[i]+tp[j] + lea r15,[32+r9] ; j=4 lea rcx,[32+rcx] adc rdx,0 mov r13,rdx @@ -975,25 +975,25 @@ ALIGN 32 $L$inner4x: - mul rbx + mul rbx ; ap[j]*bp[i] add r10,rax mov rax,QWORD[((-16))+rcx] adc rdx,0 - add r10,QWORD[16+r14] + add r10,QWORD[16+r14] ; ap[j]*bp[i]+tp[j] lea r14,[32+r14] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+r15*1+rsi] adc rdx,0 add r13,r10 adc rdx,0 - mov QWORD[((-32))+r14],rdi + mov QWORD[((-32))+r14],rdi ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[((-8))+rcx] adc rdx,0 @@ -1001,33 +1001,33 @@ adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[r15*1+rsi] adc rdx,0 add rdi,r11 adc rdx,0 - mov QWORD[((-24))+r14],r13 + mov QWORD[((-24))+r14],r13 ; tp[j-1] mov r13,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r10,rax mov rax,QWORD[rcx] adc rdx,0 - add r10,QWORD[r14] + add r10,QWORD[r14] ; ap[j]*bp[i]+tp[j] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[8+r15*1+rsi] adc rdx,0 add r13,r10 adc rdx,0 - mov QWORD[((-16))+r14],rdi + mov QWORD[((-16))+r14],rdi ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,QWORD[8+rcx] adc rdx,0 @@ -1035,38 +1035,38 @@ adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax mov rax,QWORD[16+r15*1+rsi] adc rdx,0 add rdi,r11 lea rcx,[32+rcx] adc rdx,0 - mov QWORD[((-8))+r14],r13 + mov QWORD[((-8))+r14],r13 ; tp[j-1] mov r13,rdx - add r15,32 + add r15,32 ; j+=4 jnz NEAR $L$inner4x - mul rbx + mul rbx ; ap[j]*bp[i] add r10,rax mov rax,QWORD[((-16))+rcx] adc rdx,0 - add r10,QWORD[16+r14] + add r10,QWORD[16+r14] ; ap[j]*bp[i]+tp[j] lea r14,[32+r14] adc rdx,0 mov r11,rdx - mul rbp + mul rbp ; np[j]*m1 add r13,rax mov rax,QWORD[((-8))+rsi] adc rdx,0 add r13,r10 adc rdx,0 - mov QWORD[((-32))+r14],rdi + mov QWORD[((-32))+r14],rdi ; tp[j-1] mov rdi,rdx - mul rbx + mul rbx ; ap[j]*bp[i] add r11,rax mov rax,rbp mov rbp,QWORD[((-8))+rcx] @@ -1075,39 +1075,39 @@ adc rdx,0 mov r10,rdx - mul rbp + mul rbp ; np[j]*m1 add rdi,rax - mov rax,QWORD[r9*1+rsi] + mov rax,QWORD[r9*1+rsi] ; ap[0] adc rdx,0 add rdi,r11 adc rdx,0 - mov QWORD[((-24))+r14],r13 + mov QWORD[((-24))+r14],r13 ; tp[j-1] mov r13,rdx - mov QWORD[((-16))+r14],rdi - lea rcx,[r9*1+rcx] + mov QWORD[((-16))+r14],rdi ; tp[j-1] + lea rcx,[r9*1+rcx] ; rewind %rcx xor rdi,rdi add r13,r10 adc rdi,0 - add r13,QWORD[r14] - adc rdi,0 + add r13,QWORD[r14] ; pull upmost overflow bit + adc rdi,0 ; upmost overflow bit mov QWORD[((-8))+r14],r13 cmp r12,QWORD[((16+8))+rsp] jb NEAR $L$outer4x xor rax,rax - sub rbp,r13 - adc r15,r15 + sub rbp,r13 ; compare top-most words + adc r15,r15 ; %r15 is zero or rdi,r15 - sub rax,rdi - lea rbx,[r9*1+r14] + sub rax,rdi ; %rax=-%rdi + lea rbx,[r9*1+r14] ; tptr in .sqr4x_sub mov r12,QWORD[rcx] - lea rbp,[rcx] + lea rbp,[rcx] ; nptr in .sqr4x_sub mov rcx,r9 sar rcx,3+2 - mov rdi,QWORD[((56+8))+rsp] - dec r12 + mov rdi,QWORD[((56+8))+rsp] ; rptr in .sqr4x_sub + dec r12 ; so that after 'not' we get -n[0] xor r10,r10 mov r13,QWORD[8+rbp] mov r14,QWORD[16+rbp] @@ -1149,35 +1149,35 @@ $L$power5_prologue: - - - - shl r9d,3 - lea r10d,[r9*2+r9] +; num is declared as an int, a 32-bit parameter, so the upper half is +; undefined. It is important that this write to %r9, which zeros the +; upper half, come before the first access. + shl r9d,3 ; convert %r9 to bytes + lea r10d,[r9*2+r9] ; 3*%r9 neg r9 - mov r8,QWORD[r8] + mov r8,QWORD[r8] ; *n0 - - - - - - - +; ############################################################# +; Ensure that stack frame doesn't alias with %rdi+3*%r9 +; modulo 4096, which covers ret[num], am[num] and n[num] +; (see bn_exp.c). This is done to allow memory disambiguation +; logic do its magic. [Extra 256 bytes is for power mask +; calculated from 7th argument, the index.] +; lea r11,[((-320))+r9*2+rsp] mov rbp,rsp sub r11,rdi and r11,4095 cmp r10,r11 jb NEAR $L$pwr_sp_alt - sub rbp,r11 - lea rbp,[((-320))+r9*2+rbp] + sub rbp,r11 ; align with %rsi + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*num*8+256) jmp NEAR $L$pwr_sp_done ALIGN 32 $L$pwr_sp_alt: lea r10,[((4096-320))+r9*2] - lea rbp,[((-320))+r9*2+rbp] + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*num*8+256) sub r11,r10 mov r10,0 cmovc r11,r10 @@ -1203,22 +1203,22 @@ mov r10,r9 neg r9 - - - - - - - - - +; ############################################################# +; Stack layout +; +; +0 saved %r9, used in reduction section +; +8 &t[2*%r9], used in reduction section +; +32 saved *n0 +; +40 saved %rsp +; +48 t[2*%r9] +; mov QWORD[32+rsp],r8 - mov QWORD[40+rsp],rax + mov QWORD[40+rsp],rax ; save original %rsp $L$power5_body: - movq xmm1,rdi - movq xmm2,rcx - movq xmm3,r10 + movq xmm1,rdi ; save %rdi, used in sqr8x + movq xmm2,rcx ; save %rcx + movq xmm3,r10 ; -%r9, used in sqr8x movq xmm4,rdx call __bn_sqr8x_internal @@ -1240,7 +1240,7 @@ call mul4x_internal - mov rsi,QWORD[40+rsp] + mov rsi,QWORD[40+rsp] ; restore %rsp mov rax,1 mov r15,QWORD[((-48))+rsi] @@ -1272,261 +1272,261 @@ __bn_sqr8x_internal: _CET_ENDBR +; ############################################################# +; Squaring part: +; +; a) multiply-n-add everything but a[i]*a[i]; +; b) shift result of a) by 1 to the left and accumulate +; a[i]*a[i] products; +; +; ############################################################# +; a[1]a[0] +; a[2]a[0] +; a[3]a[0] +; a[2]a[1] +; a[4]a[0] +; a[3]a[1] +; a[5]a[0] +; a[4]a[1] +; a[3]a[2] +; a[6]a[0] +; a[5]a[1] +; a[4]a[2] +; a[7]a[0] +; a[6]a[1] +; a[5]a[2] +; a[4]a[3] +; a[7]a[1] +; a[6]a[2] +; a[5]a[3] +; a[7]a[2] +; a[6]a[3] +; a[5]a[4] +; a[7]a[3] +; a[6]a[4] +; a[7]a[4] +; a[6]a[5] +; a[7]a[5] +; a[7]a[6] +; a[1]a[0] +; a[2]a[0] +; a[3]a[0] +; a[4]a[0] +; a[5]a[0] +; a[6]a[0] +; a[7]a[0] +; a[2]a[1] +; a[3]a[1] +; a[4]a[1] +; a[5]a[1] +; a[6]a[1] +; a[7]a[1] +; a[3]a[2] +; a[4]a[2] +; a[5]a[2] +; a[6]a[2] +; a[7]a[2] +; a[4]a[3] +; a[5]a[3] +; a[6]a[3] +; a[7]a[3] +; a[5]a[4] +; a[6]a[4] +; a[7]a[4] +; a[6]a[5] +; a[7]a[5] +; a[7]a[6] +; a[0]a[0] +; a[1]a[1] +; a[2]a[2] +; a[3]a[3] +; a[4]a[4] +; a[5]a[5] +; a[6]a[6] +; a[7]a[7] + lea rbp,[32+r10] ; %rbp=-(%r9-32) + lea rsi,[r9*1+rsi] ; end of a[] buffer, (%rsi,%rbp)=&ap[2] + mov rcx,r9 ; %rcx=%r9 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - lea rbp,[32+r10] - lea rsi,[r9*1+rsi] - - mov rcx,r9 - - - mov r14,QWORD[((-32))+rbp*1+rsi] - lea rdi,[((48+8))+r9*2+rsp] - mov rax,QWORD[((-24))+rbp*1+rsi] - lea rdi,[((-32))+rbp*1+rdi] - mov rbx,QWORD[((-16))+rbp*1+rsi] +; comments apply to %r9==8 case + mov r14,QWORD[((-32))+rbp*1+rsi] ; a[0] + lea rdi,[((48+8))+r9*2+rsp] ; end of tp[] buffer, &tp[2*%r9] + mov rax,QWORD[((-24))+rbp*1+rsi] ; a[1] + lea rdi,[((-32))+rbp*1+rdi] ; end of tp[] window, &tp[2*%r9-"%rbp"] + mov rbx,QWORD[((-16))+rbp*1+rsi] ; a[2] mov r15,rax - mul r14 - mov r10,rax - mov rax,rbx + mul r14 ; a[1]*a[0] + mov r10,rax ; a[1]*a[0] + mov rax,rbx ; a[2] mov r11,rdx - mov QWORD[((-24))+rbp*1+rdi],r10 + mov QWORD[((-24))+rbp*1+rdi],r10 ; t[1] - mul r14 + mul r14 ; a[2]*a[0] add r11,rax mov rax,rbx adc rdx,0 - mov QWORD[((-16))+rbp*1+rdi],r11 + mov QWORD[((-16))+rbp*1+rdi],r11 ; t[2] mov r10,rdx - mov rbx,QWORD[((-8))+rbp*1+rsi] - mul r15 - mov r12,rax + mov rbx,QWORD[((-8))+rbp*1+rsi] ; a[3] + mul r15 ; a[2]*a[1] + mov r12,rax ; a[2]*a[1]+t[3] mov rax,rbx mov r13,rdx lea rcx,[rbp] - mul r14 - add r10,rax + mul r14 ; a[3]*a[0] + add r10,rax ; a[3]*a[0]+a[2]*a[1]+t[3] mov rax,rbx mov r11,rdx adc r11,0 add r10,r12 adc r11,0 - mov QWORD[((-8))+rcx*1+rdi],r10 + mov QWORD[((-8))+rcx*1+rdi],r10 ; t[3] jmp NEAR $L$sqr4x_1st ALIGN 32 $L$sqr4x_1st: - mov rbx,QWORD[rcx*1+rsi] - mul r15 - add r13,rax + mov rbx,QWORD[rcx*1+rsi] ; a[4] + mul r15 ; a[3]*a[1] + add r13,rax ; a[3]*a[1]+t[4] mov rax,rbx mov r12,rdx adc r12,0 - mul r14 - add r11,rax - mov rax,rbx - mov rbx,QWORD[8+rcx*1+rsi] + mul r14 ; a[4]*a[0] + add r11,rax ; a[4]*a[0]+a[3]*a[1]+t[4] + mov rax,rbx ; a[3] + mov rbx,QWORD[8+rcx*1+rsi] ; a[5] mov r10,rdx adc r10,0 add r11,r13 adc r10,0 - mul r15 - add r12,rax + mul r15 ; a[4]*a[3] + add r12,rax ; a[4]*a[3]+t[5] mov rax,rbx - mov QWORD[rcx*1+rdi],r11 + mov QWORD[rcx*1+rdi],r11 ; t[4] mov r13,rdx adc r13,0 - mul r14 - add r10,rax + mul r14 ; a[5]*a[2] + add r10,rax ; a[5]*a[2]+a[4]*a[3]+t[5] mov rax,rbx - mov rbx,QWORD[16+rcx*1+rsi] + mov rbx,QWORD[16+rcx*1+rsi] ; a[6] mov r11,rdx adc r11,0 add r10,r12 adc r11,0 - mul r15 - add r13,rax + mul r15 ; a[5]*a[3] + add r13,rax ; a[5]*a[3]+t[6] mov rax,rbx - mov QWORD[8+rcx*1+rdi],r10 + mov QWORD[8+rcx*1+rdi],r10 ; t[5] mov r12,rdx adc r12,0 - mul r14 - add r11,rax - mov rax,rbx - mov rbx,QWORD[24+rcx*1+rsi] + mul r14 ; a[6]*a[2] + add r11,rax ; a[6]*a[2]+a[5]*a[3]+t[6] + mov rax,rbx ; a[3] + mov rbx,QWORD[24+rcx*1+rsi] ; a[7] mov r10,rdx adc r10,0 add r11,r13 adc r10,0 - mul r15 - add r12,rax + mul r15 ; a[6]*a[5] + add r12,rax ; a[6]*a[5]+t[7] mov rax,rbx - mov QWORD[16+rcx*1+rdi],r11 + mov QWORD[16+rcx*1+rdi],r11 ; t[6] mov r13,rdx adc r13,0 lea rcx,[32+rcx] - mul r14 - add r10,rax + mul r14 ; a[7]*a[4] + add r10,rax ; a[7]*a[4]+a[6]*a[5]+t[6] mov rax,rbx mov r11,rdx adc r11,0 add r10,r12 adc r11,0 - mov QWORD[((-8))+rcx*1+rdi],r10 + mov QWORD[((-8))+rcx*1+rdi],r10 ; t[7] cmp rcx,0 jne NEAR $L$sqr4x_1st - mul r15 + mul r15 ; a[7]*a[5] add r13,rax lea rbp,[16+rbp] adc rdx,0 add r13,r11 adc rdx,0 - mov QWORD[rdi],r13 + mov QWORD[rdi],r13 ; t[8] mov r12,rdx - mov QWORD[8+rdi],rdx + mov QWORD[8+rdi],rdx ; t[9] jmp NEAR $L$sqr4x_outer ALIGN 32 -$L$sqr4x_outer: - mov r14,QWORD[((-32))+rbp*1+rsi] - lea rdi,[((48+8))+r9*2+rsp] - mov rax,QWORD[((-24))+rbp*1+rsi] - lea rdi,[((-32))+rbp*1+rdi] - mov rbx,QWORD[((-16))+rbp*1+rsi] +$L$sqr4x_outer: ; comments apply to %r9==6 case + mov r14,QWORD[((-32))+rbp*1+rsi] ; a[0] + lea rdi,[((48+8))+r9*2+rsp] ; end of tp[] buffer, &tp[2*%r9] + mov rax,QWORD[((-24))+rbp*1+rsi] ; a[1] + lea rdi,[((-32))+rbp*1+rdi] ; end of tp[] window, &tp[2*%r9-"%rbp"] + mov rbx,QWORD[((-16))+rbp*1+rsi] ; a[2] mov r15,rax - mul r14 - mov r10,QWORD[((-24))+rbp*1+rdi] - add r10,rax - mov rax,rbx + mul r14 ; a[1]*a[0] + mov r10,QWORD[((-24))+rbp*1+rdi] ; t[1] + add r10,rax ; a[1]*a[0]+t[1] + mov rax,rbx ; a[2] adc rdx,0 - mov QWORD[((-24))+rbp*1+rdi],r10 + mov QWORD[((-24))+rbp*1+rdi],r10 ; t[1] mov r11,rdx - mul r14 + mul r14 ; a[2]*a[0] add r11,rax mov rax,rbx adc rdx,0 - add r11,QWORD[((-16))+rbp*1+rdi] + add r11,QWORD[((-16))+rbp*1+rdi] ; a[2]*a[0]+t[2] mov r10,rdx adc r10,0 - mov QWORD[((-16))+rbp*1+rdi],r11 + mov QWORD[((-16))+rbp*1+rdi],r11 ; t[2] xor r12,r12 - mov rbx,QWORD[((-8))+rbp*1+rsi] - mul r15 - add r12,rax + mov rbx,QWORD[((-8))+rbp*1+rsi] ; a[3] + mul r15 ; a[2]*a[1] + add r12,rax ; a[2]*a[1]+t[3] mov rax,rbx adc rdx,0 add r12,QWORD[((-8))+rbp*1+rdi] mov r13,rdx adc r13,0 - mul r14 - add r10,rax + mul r14 ; a[3]*a[0] + add r10,rax ; a[3]*a[0]+a[2]*a[1]+t[3] mov rax,rbx adc rdx,0 add r10,r12 mov r11,rdx adc r11,0 - mov QWORD[((-8))+rbp*1+rdi],r10 + mov QWORD[((-8))+rbp*1+rdi],r10 ; t[3] lea rcx,[rbp] jmp NEAR $L$sqr4x_inner ALIGN 32 $L$sqr4x_inner: - mov rbx,QWORD[rcx*1+rsi] - mul r15 - add r13,rax + mov rbx,QWORD[rcx*1+rsi] ; a[4] + mul r15 ; a[3]*a[1] + add r13,rax ; a[3]*a[1]+t[4] mov rax,rbx mov r12,rdx adc r12,0 @@ -1534,254 +1534,254 @@ adc r12,0 DB 0x67 - mul r14 - add r11,rax - mov rax,rbx - mov rbx,QWORD[8+rcx*1+rsi] + mul r14 ; a[4]*a[0] + add r11,rax ; a[4]*a[0]+a[3]*a[1]+t[4] + mov rax,rbx ; a[3] + mov rbx,QWORD[8+rcx*1+rsi] ; a[5] mov r10,rdx adc r10,0 add r11,r13 adc r10,0 - mul r15 - add r12,rax - mov QWORD[rcx*1+rdi],r11 + mul r15 ; a[4]*a[3] + add r12,rax ; a[4]*a[3]+t[5] + mov QWORD[rcx*1+rdi],r11 ; t[4] mov rax,rbx mov r13,rdx adc r13,0 add r12,QWORD[8+rcx*1+rdi] - lea rcx,[16+rcx] + lea rcx,[16+rcx] ; j++ adc r13,0 - mul r14 - add r10,rax + mul r14 ; a[5]*a[2] + add r10,rax ; a[5]*a[2]+a[4]*a[3]+t[5] mov rax,rbx adc rdx,0 add r10,r12 mov r11,rdx adc r11,0 - mov QWORD[((-8))+rcx*1+rdi],r10 + mov QWORD[((-8))+rcx*1+rdi],r10 ; t[5], "preloaded t[1]" below cmp rcx,0 jne NEAR $L$sqr4x_inner DB 0x67 - mul r15 + mul r15 ; a[5]*a[3] add r13,rax adc rdx,0 add r13,r11 adc rdx,0 - mov QWORD[rdi],r13 + mov QWORD[rdi],r13 ; t[6], "preloaded t[2]" below mov r12,rdx - mov QWORD[8+rdi],rdx + mov QWORD[8+rdi],rdx ; t[7], "preloaded t[3]" below add rbp,16 jnz NEAR $L$sqr4x_outer - - mov r14,QWORD[((-32))+rsi] - lea rdi,[((48+8))+r9*2+rsp] - mov rax,QWORD[((-24))+rsi] - lea rdi,[((-32))+rbp*1+rdi] - mov rbx,QWORD[((-16))+rsi] +; comments apply to %r9==4 case + mov r14,QWORD[((-32))+rsi] ; a[0] + lea rdi,[((48+8))+r9*2+rsp] ; end of tp[] buffer, &tp[2*%r9] + mov rax,QWORD[((-24))+rsi] ; a[1] + lea rdi,[((-32))+rbp*1+rdi] ; end of tp[] window, &tp[2*%r9-"%rbp"] + mov rbx,QWORD[((-16))+rsi] ; a[2] mov r15,rax - mul r14 - add r10,rax - mov rax,rbx + mul r14 ; a[1]*a[0] + add r10,rax ; a[1]*a[0]+t[1], preloaded t[1] + mov rax,rbx ; a[2] mov r11,rdx adc r11,0 - mul r14 + mul r14 ; a[2]*a[0] add r11,rax mov rax,rbx - mov QWORD[((-24))+rdi],r10 + mov QWORD[((-24))+rdi],r10 ; t[1] mov r10,rdx adc r10,0 - add r11,r13 - mov rbx,QWORD[((-8))+rsi] + add r11,r13 ; a[2]*a[0]+t[2], preloaded t[2] + mov rbx,QWORD[((-8))+rsi] ; a[3] adc r10,0 - mul r15 - add r12,rax + mul r15 ; a[2]*a[1] + add r12,rax ; a[2]*a[1]+t[3], preloaded t[3] mov rax,rbx - mov QWORD[((-16))+rdi],r11 + mov QWORD[((-16))+rdi],r11 ; t[2] mov r13,rdx adc r13,0 - mul r14 - add r10,rax + mul r14 ; a[3]*a[0] + add r10,rax ; a[3]*a[0]+a[2]*a[1]+t[3] mov rax,rbx mov r11,rdx adc r11,0 add r10,r12 adc r11,0 - mov QWORD[((-8))+rdi],r10 + mov QWORD[((-8))+rdi],r10 ; t[3] - mul r15 + mul r15 ; a[3]*a[1] add r13,rax - mov rax,QWORD[((-16))+rsi] + mov rax,QWORD[((-16))+rsi] ; a[2] adc rdx,0 add r13,r11 adc rdx,0 - mov QWORD[rdi],r13 + mov QWORD[rdi],r13 ; t[4] mov r12,rdx - mov QWORD[8+rdi],rdx + mov QWORD[8+rdi],rdx ; t[5] - mul rbx + mul rbx ; a[2]*a[3] add rbp,16 xor r14,r14 - sub rbp,r9 + sub rbp,r9 ; %rbp=16-%r9 xor r15,r15 - add rax,r12 + add rax,r12 ; t[5] adc rdx,0 - mov QWORD[8+rdi],rax - mov QWORD[16+rdi],rdx - mov QWORD[24+rdi],r15 + mov QWORD[8+rdi],rax ; t[5] + mov QWORD[16+rdi],rdx ; t[6] + mov QWORD[24+rdi],r15 ; t[7] - mov rax,QWORD[((-16))+rbp*1+rsi] + mov rax,QWORD[((-16))+rbp*1+rsi] ; a[0] lea rdi,[((48+8))+rsp] - xor r10,r10 - mov r11,QWORD[8+rdi] + xor r10,r10 ; t[0] + mov r11,QWORD[8+rdi] ; t[1] - lea r12,[r10*2+r14] + lea r12,[r10*2+r14] ; t[2*i]<<1 | shift shr r10,63 - lea r13,[r11*2+rcx] + lea r13,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r13,r10 - mov r10,QWORD[16+rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[24+rdi] + or r13,r10 ; | t[2*i]>>63 + mov r10,QWORD[16+rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[24+rdi] ; t[2*i+2+1] # prefetch adc r12,rax - mov rax,QWORD[((-8))+rbp*1+rsi] + mov rax,QWORD[((-8))+rbp*1+rsi] ; a[i+1] # prefetch mov QWORD[rdi],r12 adc r13,rdx - lea rbx,[r10*2+r14] + lea rbx,[r10*2+r14] ; t[2*i]<<1 | shift mov QWORD[8+rdi],r13 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 shr r10,63 - lea r8,[r11*2+rcx] + lea r8,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r8,r10 - mov r10,QWORD[32+rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[40+rdi] + or r8,r10 ; | t[2*i]>>63 + mov r10,QWORD[32+rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[40+rdi] ; t[2*i+2+1] # prefetch adc rbx,rax - mov rax,QWORD[rbp*1+rsi] + mov rax,QWORD[rbp*1+rsi] ; a[i+1] # prefetch mov QWORD[16+rdi],rbx adc r8,rdx lea rbp,[16+rbp] mov QWORD[24+rdi],r8 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 lea rdi,[64+rdi] jmp NEAR $L$sqr4x_shift_n_add ALIGN 32 $L$sqr4x_shift_n_add: - lea r12,[r10*2+r14] + lea r12,[r10*2+r14] ; t[2*i]<<1 | shift shr r10,63 - lea r13,[r11*2+rcx] + lea r13,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r13,r10 - mov r10,QWORD[((-16))+rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[((-8))+rdi] + or r13,r10 ; | t[2*i]>>63 + mov r10,QWORD[((-16))+rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[((-8))+rdi] ; t[2*i+2+1] # prefetch adc r12,rax - mov rax,QWORD[((-8))+rbp*1+rsi] + mov rax,QWORD[((-8))+rbp*1+rsi] ; a[i+1] # prefetch mov QWORD[((-32))+rdi],r12 adc r13,rdx - lea rbx,[r10*2+r14] + lea rbx,[r10*2+r14] ; t[2*i]<<1 | shift mov QWORD[((-24))+rdi],r13 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 shr r10,63 - lea r8,[r11*2+rcx] + lea r8,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r8,r10 - mov r10,QWORD[rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[8+rdi] + or r8,r10 ; | t[2*i]>>63 + mov r10,QWORD[rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[8+rdi] ; t[2*i+2+1] # prefetch adc rbx,rax - mov rax,QWORD[rbp*1+rsi] + mov rax,QWORD[rbp*1+rsi] ; a[i+1] # prefetch mov QWORD[((-16))+rdi],rbx adc r8,rdx - lea r12,[r10*2+r14] + lea r12,[r10*2+r14] ; t[2*i]<<1 | shift mov QWORD[((-8))+rdi],r8 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 shr r10,63 - lea r13,[r11*2+rcx] + lea r13,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r13,r10 - mov r10,QWORD[16+rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[24+rdi] + or r13,r10 ; | t[2*i]>>63 + mov r10,QWORD[16+rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[24+rdi] ; t[2*i+2+1] # prefetch adc r12,rax - mov rax,QWORD[8+rbp*1+rsi] + mov rax,QWORD[8+rbp*1+rsi] ; a[i+1] # prefetch mov QWORD[rdi],r12 adc r13,rdx - lea rbx,[r10*2+r14] + lea rbx,[r10*2+r14] ; t[2*i]<<1 | shift mov QWORD[8+rdi],r13 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 shr r10,63 - lea r8,[r11*2+rcx] + lea r8,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r8,r10 - mov r10,QWORD[32+rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[40+rdi] + or r8,r10 ; | t[2*i]>>63 + mov r10,QWORD[32+rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[40+rdi] ; t[2*i+2+1] # prefetch adc rbx,rax - mov rax,QWORD[16+rbp*1+rsi] + mov rax,QWORD[16+rbp*1+rsi] ; a[i+1] # prefetch mov QWORD[16+rdi],rbx adc r8,rdx mov QWORD[24+rdi],r8 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 lea rdi,[64+rdi] add rbp,32 jnz NEAR $L$sqr4x_shift_n_add - lea r12,[r10*2+r14] + lea r12,[r10*2+r14] ; t[2*i]<<1 | shift DB 0x67 shr r10,63 - lea r13,[r11*2+rcx] + lea r13,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r13,r10 - mov r10,QWORD[((-16))+rdi] - mov r14,r11 - mul rax - neg r15 - mov r11,QWORD[((-8))+rdi] + or r13,r10 ; | t[2*i]>>63 + mov r10,QWORD[((-16))+rdi] ; t[2*i+2] # prefetch + mov r14,r11 ; shift=t[2*i+1]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf + mov r11,QWORD[((-8))+rdi] ; t[2*i+2+1] # prefetch adc r12,rax - mov rax,QWORD[((-8))+rsi] + mov rax,QWORD[((-8))+rsi] ; a[i+1] # prefetch mov QWORD[((-32))+rdi],r12 adc r13,rdx - lea rbx,[r10*2+r14] + lea rbx,[r10*2+r14] ; t[2*i]<<1|shift mov QWORD[((-24))+rdi],r13 - sbb r15,r15 + sbb r15,r15 ; mov cf,%r15 shr r10,63 - lea r8,[r11*2+rcx] + lea r8,[r11*2+rcx] ; t[2*i+1]<<1 | shr r11,63 - or r8,r10 - mul rax - neg r15 + or r8,r10 ; | t[2*i]>>63 + mul rax ; a[i]*a[i] + neg r15 ; mov %r15,cf adc rbx,rax adc r8,rdx mov QWORD[((-16))+rdi],rbx @@ -1789,17 +1789,17 @@ movq rbp,xmm2 __bn_sqr8x_reduction: xor rax,rax - lea rcx,[rbp*1+r9] - lea rdx,[((48+8))+r9*2+rsp] + lea rcx,[rbp*1+r9] ; end of n[] + lea rdx,[((48+8))+r9*2+rsp] ; end of t[] buffer mov QWORD[((0+8))+rsp],rcx - lea rdi,[((48+8))+r9*1+rsp] + lea rdi,[((48+8))+r9*1+rsp] ; end of initial t[] window mov QWORD[((8+8))+rsp],rdx neg r9 jmp NEAR $L$8x_reduction_loop ALIGN 32 $L$8x_reduction_loop: - lea rdi,[r9*1+rdi] + lea rdi,[r9*1+rdi] ; start of current t[] window DB 0x66 mov rbx,QWORD[rdi] mov r9,QWORD[8+rdi] @@ -1809,20 +1809,20 @@ mov r13,QWORD[40+rdi] mov r14,QWORD[48+rdi] mov r15,QWORD[56+rdi] - mov QWORD[rdx],rax + mov QWORD[rdx],rax ; store top-most carry bit lea rdi,[64+rdi] DB 0x67 mov r8,rbx - imul rbx,QWORD[((32+8))+rsp] - mov rax,QWORD[rbp] + imul rbx,QWORD[((32+8))+rsp] ; n0*a[0] + mov rax,QWORD[rbp] ; n[0] mov ecx,8 jmp NEAR $L$8x_reduce ALIGN 32 $L$8x_reduce: mul rbx - mov rax,QWORD[8+rbp] + mov rax,QWORD[8+rbp] ; n[1] neg r8 mov r8,rdx adc r8,0 @@ -1832,7 +1832,7 @@ mov rax,QWORD[16+rbp] adc rdx,0 add r8,r9 - mov QWORD[((48-8+8))+rcx*8+rsp],rbx + mov QWORD[((48-8+8))+rcx*8+rsp],rbx ; put aside n0*a[i] mov r9,rdx adc r9,0 @@ -1841,7 +1841,7 @@ mov rax,QWORD[24+rbp] adc rdx,0 add r9,r10 - mov rsi,QWORD[((32+8))+rsp] + mov rsi,QWORD[((32+8))+rsp] ; pull n0, borrow %rsi mov r10,rdx adc r10,0 @@ -1849,7 +1849,7 @@ add r11,rax mov rax,QWORD[32+rbp] adc rdx,0 - imul rsi,r8 + imul rsi,r8 ; modulo-scheduled add r10,r11 mov r11,rdx adc r11,0 @@ -1879,9 +1879,9 @@ adc r14,0 mul rbx - mov rbx,rsi + mov rbx,rsi ; n0*a[i] add r15,rax - mov rax,QWORD[rbp] + mov rax,QWORD[rbp] ; n[0] adc rdx,0 add r14,r15 mov r15,rdx @@ -1892,8 +1892,8 @@ lea rbp,[64+rbp] xor rax,rax - mov rdx,QWORD[((8+8))+rsp] - cmp rbp,QWORD[((0+8))+rsp] + mov rdx,QWORD[((8+8))+rsp] ; pull end of t[] + cmp rbp,QWORD[((0+8))+rsp] ; end of n[]? jae NEAR $L$8x_no_tail DB 0x66 @@ -1905,9 +1905,9 @@ adc r13,QWORD[40+rdi] adc r14,QWORD[48+rdi] adc r15,QWORD[56+rdi] - sbb rsi,rsi + sbb rsi,rsi ; top carry - mov rbx,QWORD[((48+56+8))+rsp] + mov rbx,QWORD[((48+56+8))+rsp] ; pull n0*a[0] mov ecx,8 mov rax,QWORD[rbp] jmp NEAR $L$8x_tail @@ -1917,7 +1917,7 @@ mul rbx add r8,rax mov rax,QWORD[8+rbp] - mov QWORD[rdi],r8 + mov QWORD[rdi],r8 ; save result mov r8,rdx adc r8,0 @@ -1926,7 +1926,7 @@ mov rax,QWORD[16+rbp] adc rdx,0 add r8,r9 - lea rdi,[8+rdi] + lea rdi,[8+rdi] ; %rdi++ mov r9,rdx adc r9,0 @@ -1971,11 +1971,11 @@ adc r14,0 mul rbx - mov rbx,QWORD[((48-16+8))+rcx*8+rsp] + mov rbx,QWORD[((48-16+8))+rcx*8+rsp] ; pull n0*a[i] add r15,rax adc rdx,0 add r14,r15 - mov rax,QWORD[rbp] + mov rax,QWORD[rbp] ; pull n[0] mov r15,rdx adc r15,0 @@ -1983,13 +1983,13 @@ jnz NEAR $L$8x_tail lea rbp,[64+rbp] - mov rdx,QWORD[((8+8))+rsp] - cmp rbp,QWORD[((0+8))+rsp] - jae NEAR $L$8x_tail_done + mov rdx,QWORD[((8+8))+rsp] ; pull end of t[] + cmp rbp,QWORD[((0+8))+rsp] ; end of n[]? + jae NEAR $L$8x_tail_done ; break out of loop - mov rbx,QWORD[((48+56+8))+rsp] + mov rbx,QWORD[((48+56+8))+rsp] ; pull n0*a[0] neg rsi - mov rax,QWORD[rbp] + mov rax,QWORD[rbp] ; pull n[0] adc r8,QWORD[rdi] adc r9,QWORD[8+rdi] adc r10,QWORD[16+rdi] @@ -1998,7 +1998,7 @@ adc r13,QWORD[40+rdi] adc r14,QWORD[48+rdi] adc r15,QWORD[56+rdi] - sbb rsi,rsi + sbb rsi,rsi ; top carry mov ecx,8 jmp NEAR $L$8x_tail @@ -2006,7 +2006,7 @@ ALIGN 32 $L$8x_tail_done: xor rax,rax - add r8,QWORD[rdx] + add r8,QWORD[rdx] ; can this overflow? adc r9,0 adc r10,0 adc r11,0 @@ -2026,15 +2026,15 @@ adc r13,QWORD[40+rdi] adc r14,QWORD[48+rdi] adc r15,QWORD[56+rdi] - adc rax,0 - mov rcx,QWORD[((-8))+rbp] + adc rax,0 ; top-most carry + mov rcx,QWORD[((-8))+rbp] ; np[num-1] xor rsi,rsi - movq rbp,xmm2 + movq rbp,xmm2 ; restore %rbp - mov QWORD[rdi],r8 + mov QWORD[rdi],r8 ; store top 512 bits mov QWORD[8+rdi],r9 - movq r9,xmm3 + movq r9,xmm3 ; %r9 is %r9, can't be moved upwards mov QWORD[16+rdi],r10 mov QWORD[24+rdi],r11 mov QWORD[32+rdi],r12 @@ -2043,7 +2043,7 @@ mov QWORD[56+rdi],r15 lea rdi,[64+rdi] - cmp rdi,rdx + cmp rdi,rdx ; end of t[]? jb NEAR $L$8x_reduction_loop ret @@ -2053,13 +2053,13 @@ __bn_post4x_internal: mov r12,QWORD[rbp] - lea rbx,[r9*1+rdi] + lea rbx,[r9*1+rdi] ; %rdi was %rbx above mov rcx,r9 - movq rdi,xmm1 + movq rdi,xmm1 ; restore %rdi neg rax - movq rsi,xmm1 + movq rsi,xmm1 ; prepare for back-to-back call sar rcx,3+2 - dec r12 + dec r12 ; so that after 'not' we get -n[0] xor r10,r10 mov r13,QWORD[8+rbp] mov r14,QWORD[16+rbp] @@ -2083,7 +2083,7 @@ and r14,rax and r15,rax - neg r10 + neg r10 ; mov %r10,%cf adc r12,QWORD[rbx] adc r13,QWORD[8+rbx] adc r14,QWORD[16+rbx] @@ -2091,16 +2091,16 @@ mov QWORD[rdi],r12 lea rbx,[32+rbx] mov QWORD[8+rdi],r13 - sbb r10,r10 + sbb r10,r10 ; mov %cf,%r10 mov QWORD[16+rdi],r14 mov QWORD[24+rdi],r15 lea rdi,[32+rdi] - inc rcx + inc rcx ; pass %cf jnz NEAR $L$sqr4x_sub - mov r10,r9 - neg r9 + mov r10,r9 ; prepare for back-to-back call + neg r9 ; restore %r9 ret @@ -2138,42 +2138,42 @@ $L$mulx4x_prologue: +; num is declared as an int, a 32-bit parameter, so the upper half is +; undefined. It is important that this write to %r9, which zeros the +; upper half, predates the first access. + shl r9d,3 ; convert %r9 to bytes + lea r10,[r9*2+r9] ; 3*%r9 in bytes + neg r9 ; -%r9 + mov r8,QWORD[r8] ; *n0 - - - shl r9d,3 - lea r10,[r9*2+r9] - neg r9 - mov r8,QWORD[r8] - - - - - - - - - - +; ############################################################# +; Ensure that stack frame doesn't alias with +3*%r9 +; modulo 4096, which covers ret[num], am[num] and n[num] +; (see bn_exp.c). This is done to allow memory disambiguation +; logic do its magic. [Extra [num] is allocated in order +; to align with bn_power5's frame, which is cleansed after +; completing exponentiation. Extra 256 bytes is for power mask +; calculated from 7th argument, the index.] +; lea r11,[((-320))+r9*2+rsp] mov rbp,rsp sub r11,rdi and r11,4095 cmp r10,r11 jb NEAR $L$mulx4xsp_alt - sub rbp,r11 - lea rbp,[((-320))+r9*2+rbp] + sub rbp,r11 ; align with + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*%r9*8+256) jmp NEAR $L$mulx4xsp_done $L$mulx4xsp_alt: lea r10,[((4096-320))+r9*2] - lea rbp,[((-320))+r9*2+rbp] + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*%r9*8+256) sub r11,r10 mov r10,0 cmovc r11,r10 sub rbp,r11 $L$mulx4xsp_done: - and rbp,-64 + and rbp,-64 ; ensure alignment mov r11,rsp sub r11,rbp and r11,-4096 @@ -2190,25 +2190,25 @@ ja NEAR $L$mulx4x_page_walk $L$mulx4x_page_walk_done: - - - - - - - - - - - - - mov QWORD[32+rsp],r8 - mov QWORD[40+rsp],rax +; ############################################################# +; Stack layout +; +0 -num +; +8 off-loaded &b[i] +; +16 end of b[num] +; +24 inner counter +; +32 saved n0 +; +40 saved %rsp +; +48 +; +56 saved rp +; +64 tmp[num+1] +; + mov QWORD[32+rsp],r8 ; save *n0 + mov QWORD[40+rsp],rax ; save original %rsp $L$mulx4x_body: call mulx4x_internal - mov rsi,QWORD[40+rsp] + mov rsi,QWORD[40+rsp] ; restore %rsp mov rax,1 @@ -2237,44 +2237,44 @@ ALIGN 32 mulx4x_internal: - mov QWORD[8+rsp],r9 + mov QWORD[8+rsp],r9 ; save -%r9 (it was in bytes) mov r10,r9 - neg r9 + neg r9 ; restore %r9 shl r9,5 - neg r10 - lea r13,[128+r9*1+rdx] + neg r10 ; restore %r9 + lea r13,[128+r9*1+rdx] ; end of powers table (+size optimization) shr r9,5+5 - movd xmm5,DWORD[56+rax] + movd xmm5,DWORD[56+rax] ; load 7th argument sub r9,1 lea rax,[$L$inc] - mov QWORD[((16+8))+rsp],r13 - mov QWORD[((24+8))+rsp],r9 - mov QWORD[((56+8))+rsp],rdi - movdqa xmm0,XMMWORD[rax] - movdqa xmm1,XMMWORD[16+rax] - lea r10,[((88-112))+r10*1+rsp] - lea rdi,[128+rdx] + mov QWORD[((16+8))+rsp],r13 ; end of b[num] + mov QWORD[((24+8))+rsp],r9 ; inner counter + mov QWORD[((56+8))+rsp],rdi ; save %rdi + movdqa xmm0,XMMWORD[rax] ; 00000001000000010000000000000000 + movdqa xmm1,XMMWORD[16+rax] ; 00000002000000020000000200000002 + lea r10,[((88-112))+r10*1+rsp] ; place the mask after tp[num+1] (+ICache optimization) + lea rdi,[128+rdx] ; size optimization - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; broadcast index movdqa xmm4,xmm1 DB 0x67 movdqa xmm2,xmm1 DB 0x67 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[112+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[128+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[144+r10],xmm2 movdqa xmm2,xmm4 @@ -2283,17 +2283,17 @@ movdqa XMMWORD[160+r10],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[176+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[192+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[208+r10],xmm2 movdqa xmm2,xmm4 @@ -2302,17 +2302,17 @@ movdqa XMMWORD[224+r10],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[240+r10],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[256+r10],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[272+r10],xmm2 movdqa xmm2,xmm4 @@ -2332,7 +2332,7 @@ pcmpeqd xmm3,xmm5 movdqa XMMWORD[336+r10],xmm2 - pand xmm0,XMMWORD[64+rdi] + pand xmm0,XMMWORD[64+rdi] ; while it's still in register pand xmm1,XMMWORD[80+rdi] pand xmm2,XMMWORD[96+rdi] movdqa XMMWORD[352+r10],xmm3 @@ -2376,41 +2376,41 @@ por xmm0,xmm2 por xmm1,xmm3 pxor xmm0,xmm1 - - pshufd xmm1,xmm0,0x4e +; Combine the upper and lower halves of %xmm0. + pshufd xmm1,xmm0,0x4e ; Swap upper and lower halves. por xmm0,xmm1 lea rdi,[256+rdi] - movq rdx,xmm0 + movq rdx,xmm0 ; bp[0] lea rbx,[((64+32+8))+rsp] mov r9,rdx - mulx rax,r8,QWORD[rsi] - mulx r12,r11,QWORD[8+rsi] + mulx rax,r8,QWORD[rsi] ; a[0]*b[0] + mulx r12,r11,QWORD[8+rsi] ; a[1]*b[0] add r11,rax - mulx r13,rax,QWORD[16+rsi] + mulx r13,rax,QWORD[16+rsi] ; ... adc r12,rax adc r13,0 mulx r14,rax,QWORD[24+rsi] mov r15,r8 - imul r8,QWORD[((32+8))+rsp] - xor rbp,rbp + imul r8,QWORD[((32+8))+rsp] ; "t[0]"*n0 + xor rbp,rbp ; cf=0, of=0 mov rdx,r8 - mov QWORD[((8+8))+rsp],rdi + mov QWORD[((8+8))+rsp],rdi ; off-load &b[i] lea rsi,[32+rsi] adcx r13,rax - adcx r14,rbp + adcx r14,rbp ; cf=0 mulx r10,rax,QWORD[rcx] - adcx r15,rax + adcx r15,rax ; discarded adox r10,r11 mulx r11,rax,QWORD[8+rcx] adcx r10,rax adox r11,r12 mulx r12,rax,QWORD[16+rcx] - mov rdi,QWORD[((24+8))+rsp] + mov rdi,QWORD[((24+8))+rsp] ; counter value mov QWORD[((-32))+rbx],r10 adcx r11,rax adox r12,r13 @@ -2418,25 +2418,25 @@ mov rdx,r9 mov QWORD[((-24))+rbx],r11 adcx r12,rax - adox r15,rbp + adox r15,rbp ; of=0 lea rcx,[32+rcx] mov QWORD[((-16))+rbx],r12 jmp NEAR $L$mulx4x_1st ALIGN 32 $L$mulx4x_1st: - adcx r15,rbp - mulx rax,r10,QWORD[rsi] + adcx r15,rbp ; cf=0, modulo-scheduled + mulx rax,r10,QWORD[rsi] ; a[4]*b[0] adcx r10,r14 - mulx r14,r11,QWORD[8+rsi] + mulx r14,r11,QWORD[8+rsi] ; a[5]*b[0] adcx r11,rax - mulx rax,r12,QWORD[16+rsi] + mulx rax,r12,QWORD[16+rsi] ; ... adcx r12,r14 mulx r14,r13,QWORD[24+rsi] DB 0x67,0x67 mov rdx,r8 adcx r13,rax - adcx r14,rbp + adcx r14,rbp ; cf=0 lea rsi,[32+rsi] lea rbx,[32+rbx] @@ -2460,21 +2460,21 @@ lea rcx,[32+rcx] mov QWORD[((-16))+rbx],r13 - dec rdi + dec rdi ; of=0, pass cf jnz NEAR $L$mulx4x_1st - mov rax,QWORD[8+rsp] - adc r15,rbp - lea rsi,[rax*1+rsi] + mov rax,QWORD[8+rsp] ; load -num + adc r15,rbp ; modulo-scheduled + lea rsi,[rax*1+rsi] ; rewind %rsi add r14,r15 - mov rdi,QWORD[((8+8))+rsp] - adc rbp,rbp + mov rdi,QWORD[((8+8))+rsp] ; re-load &b[i] + adc rbp,rbp ; top-most carry mov QWORD[((-8))+rbx],r14 jmp NEAR $L$mulx4x_outer ALIGN 32 $L$mulx4x_outer: - lea r10,[((16-256))+rbx] + lea r10,[((16-256))+rbx] ; where 256-byte mask is (+density control) pxor xmm4,xmm4 DB 0x67,0x67 pxor xmm5,xmm5 @@ -2527,41 +2527,41 @@ por xmm4,xmm2 por xmm5,xmm3 por xmm4,xmm5 - - pshufd xmm0,xmm4,0x4e +; Combine the upper and lower halves of %xmm4 as %xmm0. + pshufd xmm0,xmm4,0x4e ; Swap upper and lower halves. por xmm0,xmm4 lea rdi,[256+rdi] - movq rdx,xmm0 + movq rdx,xmm0 ; m0=bp[i] - mov QWORD[rbx],rbp - lea rbx,[32+rax*1+rbx] - mulx r11,r8,QWORD[rsi] - xor rbp,rbp + mov QWORD[rbx],rbp ; save top-most carry + lea rbx,[32+rax*1+rbx] ; rewind %rbx + mulx r11,r8,QWORD[rsi] ; a[0]*b[i] + xor rbp,rbp ; cf=0, of=0 mov r9,rdx - mulx r12,r14,QWORD[8+rsi] - adox r8,QWORD[((-32))+rbx] + mulx r12,r14,QWORD[8+rsi] ; a[1]*b[i] + adox r8,QWORD[((-32))+rbx] ; +t[0] adcx r11,r14 - mulx r13,r15,QWORD[16+rsi] + mulx r13,r15,QWORD[16+rsi] ; ... adox r11,QWORD[((-24))+rbx] adcx r12,r15 mulx r14,rdx,QWORD[24+rsi] adox r12,QWORD[((-16))+rbx] adcx r13,rdx - lea rcx,[rax*1+rcx] + lea rcx,[rax*1+rcx] ; rewind %rcx lea rsi,[32+rsi] adox r13,QWORD[((-8))+rbx] adcx r14,rbp adox r14,rbp mov r15,r8 - imul r8,QWORD[((32+8))+rsp] + imul r8,QWORD[((32+8))+rsp] ; "t[0]"*n0 mov rdx,r8 - xor rbp,rbp - mov QWORD[((8+8))+rsp],rdi + xor rbp,rbp ; cf=0, of=0 + mov QWORD[((8+8))+rsp],rdi ; off-load &b[i] mulx r10,rax,QWORD[rcx] - adcx r15,rax + adcx r15,rax ; discarded adox r10,r11 mulx r11,rax,QWORD[8+rcx] adcx r10,rax @@ -2571,24 +2571,24 @@ adox r12,r13 mulx r15,rax,QWORD[24+rcx] mov rdx,r9 - mov rdi,QWORD[((24+8))+rsp] + mov rdi,QWORD[((24+8))+rsp] ; counter value mov QWORD[((-32))+rbx],r10 adcx r12,rax mov QWORD[((-24))+rbx],r11 - adox r15,rbp + adox r15,rbp ; of=0 mov QWORD[((-16))+rbx],r12 lea rcx,[32+rcx] jmp NEAR $L$mulx4x_inner ALIGN 32 $L$mulx4x_inner: - mulx rax,r10,QWORD[rsi] - adcx r15,rbp + mulx rax,r10,QWORD[rsi] ; a[4]*b[i] + adcx r15,rbp ; cf=0, modulo-scheduled adox r10,r14 - mulx r14,r11,QWORD[8+rsi] + mulx r14,r11,QWORD[8+rsi] ; a[5]*b[i] adcx r10,QWORD[rbx] adox r11,rax - mulx rax,r12,QWORD[16+rsi] + mulx rax,r12,QWORD[16+rsi] ; ... adcx r11,QWORD[8+rbx] adox r12,r14 mulx r14,r13,QWORD[24+rsi] @@ -2596,10 +2596,10 @@ adcx r12,QWORD[16+rbx] adox r13,rax adcx r13,QWORD[24+rbx] - adox r14,rbp + adox r14,rbp ; of=0 lea rsi,[32+rsi] lea rbx,[32+rbx] - adcx r14,rbp + adcx r14,rbp ; cf=0 adox r10,r15 mulx r15,rax,QWORD[rcx] @@ -2621,17 +2621,17 @@ adox r15,rbp mov QWORD[((-16))+rbx],r13 - dec rdi + dec rdi ; of=0, pass cf jnz NEAR $L$mulx4x_inner - mov rax,QWORD[((0+8))+rsp] - adc r15,rbp - sub rdi,QWORD[rbx] - mov rdi,QWORD[((8+8))+rsp] + mov rax,QWORD[((0+8))+rsp] ; load -num + adc r15,rbp ; modulo-scheduled + sub rdi,QWORD[rbx] ; pull top-most carry to %cf + mov rdi,QWORD[((8+8))+rsp] ; re-load &b[i] mov r10,QWORD[((16+8))+rsp] adc r14,r15 - lea rsi,[rax*1+rsi] - adc rbp,rbp + lea rsi,[rax*1+rsi] ; rewind %rsi + adc rbp,rbp ; top-most carry mov QWORD[((-8))+rbx],r14 cmp rdi,r10 @@ -2640,23 +2640,23 @@ mov r10,QWORD[((-8))+rcx] mov r8,rbp mov r12,QWORD[rax*1+rcx] - lea rbp,[rax*1+rcx] + lea rbp,[rax*1+rcx] ; rewind %rcx mov rcx,rax - lea rdi,[rax*1+rbx] + lea rdi,[rax*1+rbx] ; rewind %rbx xor eax,eax xor r15,r15 - sub r10,r14 + sub r10,r14 ; compare top-most words adc r15,r15 or r8,r15 sar rcx,3+2 - sub rax,r8 - mov rdx,QWORD[((56+8))+rsp] - dec r12 + sub rax,r8 ; %rax=-%r8 + mov rdx,QWORD[((56+8))+rsp] ; restore rp + dec r12 ; so that after 'not' we get -n[0] mov r13,QWORD[8+rbp] xor r8,r8 mov r14,QWORD[16+rbp] mov r15,QWORD[24+rbp] - jmp NEAR $L$sqrx4x_sub_entry + jmp NEAR $L$sqrx4x_sub_entry ; common post-condition global bn_powerx5 @@ -2693,35 +2693,35 @@ $L$powerx5_prologue: - - - - shl r9d,3 - lea r10,[r9*2+r9] +; num is declared as an int, a 32-bit parameter, so the upper half is +; undefined. It is important that this write to %r9, which zeros the +; upper half, predates the first access. + shl r9d,3 ; convert %r9 to bytes + lea r10,[r9*2+r9] ; 3*%r9 in bytes neg r9 - mov r8,QWORD[r8] + mov r8,QWORD[r8] ; *n0 - - - - - - - +; ############################################################# +; Ensure that stack frame doesn't alias with %rdi+3*%r9 +; modulo 4096, which covers ret[num], am[num] and n[num] +; (see bn_exp.c). This is done to allow memory disambiguation +; logic do its magic. [Extra 256 bytes is for power mask +; calculated from 7th argument, the index.] +; lea r11,[((-320))+r9*2+rsp] mov rbp,rsp sub r11,rdi and r11,4095 cmp r10,r11 jb NEAR $L$pwrx_sp_alt - sub rbp,r11 - lea rbp,[((-320))+r9*2+rbp] + sub rbp,r11 ; align with %rsi + lea rbp,[((-320))+r9*2+rbp] ; future alloca(frame+2*%r9*8+256) jmp NEAR $L$pwrx_sp_done ALIGN 32 $L$pwrx_sp_alt: lea r10,[((4096-320))+r9*2] - lea rbp,[((-320))+r9*2+rbp] + lea rbp,[((-320))+r9*2+rbp] ; alloca(frame+2*%r9*8+256) sub r11,r10 mov r10,0 cmovc r11,r10 @@ -2747,24 +2747,24 @@ mov r10,r9 neg r9 - - - - - - - - - - - +; ############################################################# +; Stack layout +; +; +0 saved %r9, used in reduction section +; +8 &t[2*%r9], used in reduction section +; +16 intermediate carry bit +; +24 top-most carry bit, used in reduction section +; +32 saved *n0 +; +40 saved %rsp +; +48 t[2*%r9] +; pxor xmm0,xmm0 - movq xmm1,rdi - movq xmm2,rcx - movq xmm3,r10 + movq xmm1,rdi ; save %rdi + movq xmm2,rcx ; save %rcx + movq xmm3,r10 ; -%r9 movq xmm4,rdx mov QWORD[32+rsp],r8 - mov QWORD[40+rsp],rax + mov QWORD[40+rsp],rax ; save original %rsp $L$powerx5_body: @@ -2779,7 +2779,7 @@ call __bn_sqrx8x_internal call __bn_postx4x_internal - mov r9,r10 + mov r9,r10 ; -num mov rdi,rsi movq rcx,xmm2 movq rdx,xmm4 @@ -2787,7 +2787,7 @@ call mulx4x_internal - mov rsi,QWORD[40+rsp] + mov rsi,QWORD[40+rsp] ; restore %rsp mov rax,1 @@ -2820,50 +2820,50 @@ __bn_sqrx8x_internal: _CET_ENDBR - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +; ################################################################# +; Squaring part: +; +; a) multiply-n-add everything but a[i]*a[i]; +; b) shift result of a) by 1 to the left and accumulate +; a[i]*a[i] products; +; +; ################################################################# +; a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] +; a[1]a[0] +; a[2]a[0] +; a[3]a[0] +; a[2]a[1] +; a[3]a[1] +; a[3]a[2] +; +; a[4]a[0] +; a[5]a[0] +; a[6]a[0] +; a[7]a[0] +; a[4]a[1] +; a[5]a[1] +; a[6]a[1] +; a[7]a[1] +; a[4]a[2] +; a[5]a[2] +; a[6]a[2] +; a[7]a[2] +; a[4]a[3] +; a[5]a[3] +; a[6]a[3] +; a[7]a[3] +; +; a[5]a[4] +; a[6]a[4] +; a[7]a[4] +; a[6]a[5] +; a[7]a[5] +; a[7]a[6] +; a[7]a[7]a[6]a[6]a[5]a[5]a[4]a[4]a[3]a[3]a[2]a[2]a[1]a[1]a[0]a[0] lea rdi,[((48+8))+rsp] lea rbp,[r9*1+rsi] - mov QWORD[((0+8))+rsp],r9 - mov QWORD[((8+8))+rsp],rbp + mov QWORD[((0+8))+rsp],r9 ; save %r9 + mov QWORD[((8+8))+rsp],rbp ; save end of %rsi jmp NEAR $L$sqr8x_zero_start ALIGN 32 @@ -2874,7 +2874,7 @@ movdqa XMMWORD[16+rdi],xmm0 movdqa XMMWORD[32+rdi],xmm0 movdqa XMMWORD[48+rdi],xmm0 -$L$sqr8x_zero_start: +$L$sqr8x_zero_start: ; aligned at 32 movdqa XMMWORD[64+rdi],xmm0 movdqa XMMWORD[80+rdi],xmm0 movdqa XMMWORD[96+rdi],xmm0 @@ -2883,8 +2883,8 @@ sub r9,64 jnz NEAR $L$sqrx8x_zero - mov rdx,QWORD[rsi] - + mov rdx,QWORD[rsi] ; a[0], modulo-scheduled +; xor %r9,%r9 # t[1], ex-%r9, zero already xor r10,r10 xor r11,r11 xor r12,r12 @@ -2892,21 +2892,21 @@ xor r14,r14 xor r15,r15 lea rdi,[((48+8))+rsp] - xor rbp,rbp + xor rbp,rbp ; cf=0, cf=0 jmp NEAR $L$sqrx8x_outer_loop ALIGN 32 $L$sqrx8x_outer_loop: - mulx rax,r8,QWORD[8+rsi] - adcx r8,r9 + mulx rax,r8,QWORD[8+rsi] ; a[1]*a[0] + adcx r8,r9 ; a[1]*a[0]+=t[1] adox r10,rax - mulx rax,r9,QWORD[16+rsi] + mulx rax,r9,QWORD[16+rsi] ; a[2]*a[0] adcx r9,r10 adox r11,rax - DB 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 + DB 0xc4,0xe2,0xab,0xf6,0x86,0x18,0x00,0x00,0x00 ; mulx 3*8(%rsi),%r10,%rax # ... adcx r10,r11 adox r12,rax - DB 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 + DB 0xc4,0xe2,0xa3,0xf6,0x86,0x20,0x00,0x00,0x00 ; mulx 4*8(%rsi),%r11,%rax adcx r11,r12 adox r13,rax mulx rax,r12,QWORD[40+rsi] @@ -2916,97 +2916,97 @@ adcx r13,r14 adox rax,r15 mulx r15,r14,QWORD[56+rsi] - mov rdx,QWORD[8+rsi] + mov rdx,QWORD[8+rsi] ; a[1] adcx r14,rax adox r15,rbp adc r15,QWORD[64+rdi] - mov QWORD[8+rdi],r8 - mov QWORD[16+rdi],r9 - sbb rcx,rcx - xor rbp,rbp + mov QWORD[8+rdi],r8 ; t[1] + mov QWORD[16+rdi],r9 ; t[2] + sbb rcx,rcx ; mov %cf,%rcx + xor rbp,rbp ; cf=0, of=0 - mulx rbx,r8,QWORD[16+rsi] - mulx rax,r9,QWORD[24+rsi] + mulx rbx,r8,QWORD[16+rsi] ; a[2]*a[1] + mulx rax,r9,QWORD[24+rsi] ; a[3]*a[1] adcx r8,r10 adox r9,rbx - mulx rbx,r10,QWORD[32+rsi] + mulx rbx,r10,QWORD[32+rsi] ; ... adcx r9,r11 adox r10,rax - DB 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 + DB 0xc4,0xe2,0xa3,0xf6,0x86,0x28,0x00,0x00,0x00 ; mulx 5*8(%rsi),%r11,%rax adcx r10,r12 adox r11,rbx - DB 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 + DB 0xc4,0xe2,0x9b,0xf6,0x9e,0x30,0x00,0x00,0x00 ; mulx 6*8(%rsi),%r12,%rbx adcx r11,r13 adox r12,r14 - DB 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 - mov rdx,QWORD[16+rsi] + DB 0xc4,0x62,0x93,0xf6,0xb6,0x38,0x00,0x00,0x00 ; mulx 7*8(%rsi),%r13,%r14 + mov rdx,QWORD[16+rsi] ; a[2] adcx r12,rax adox r13,rbx adcx r13,r15 - adox r14,rbp - adcx r14,rbp + adox r14,rbp ; of=0 + adcx r14,rbp ; cf=0 - mov QWORD[24+rdi],r8 - mov QWORD[32+rdi],r9 + mov QWORD[24+rdi],r8 ; t[3] + mov QWORD[32+rdi],r9 ; t[4] - mulx rbx,r8,QWORD[24+rsi] - mulx rax,r9,QWORD[32+rsi] + mulx rbx,r8,QWORD[24+rsi] ; a[3]*a[2] + mulx rax,r9,QWORD[32+rsi] ; a[4]*a[2] adcx r8,r10 adox r9,rbx - mulx rbx,r10,QWORD[40+rsi] + mulx rbx,r10,QWORD[40+rsi] ; ... adcx r9,r11 adox r10,rax - DB 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 + DB 0xc4,0xe2,0xa3,0xf6,0x86,0x30,0x00,0x00,0x00 ; mulx 6*8(%rsi),%r11,%rax adcx r10,r12 adox r11,r13 - DB 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 + DB 0xc4,0x62,0x9b,0xf6,0xae,0x38,0x00,0x00,0x00 ; mulx 7*8(%rsi),%r12,%r13 DB 0x3e - mov rdx,QWORD[24+rsi] + mov rdx,QWORD[24+rsi] ; a[3] adcx r11,rbx adox r12,rax adcx r12,r14 - mov QWORD[40+rdi],r8 - mov QWORD[48+rdi],r9 - mulx rax,r8,QWORD[32+rsi] - adox r13,rbp - adcx r13,rbp + mov QWORD[40+rdi],r8 ; t[5] + mov QWORD[48+rdi],r9 ; t[6] + mulx rax,r8,QWORD[32+rsi] ; a[4]*a[3] + adox r13,rbp ; of=0 + adcx r13,rbp ; cf=0 - mulx rbx,r9,QWORD[40+rsi] + mulx rbx,r9,QWORD[40+rsi] ; a[5]*a[3] adcx r8,r10 adox r9,rax - mulx rax,r10,QWORD[48+rsi] + mulx rax,r10,QWORD[48+rsi] ; ... adcx r9,r11 adox r10,r12 mulx r12,r11,QWORD[56+rsi] - mov rdx,QWORD[32+rsi] - mov r14,QWORD[40+rsi] + mov rdx,QWORD[32+rsi] ; a[4] + mov r14,QWORD[40+rsi] ; a[5] adcx r10,rbx adox r11,rax - mov r15,QWORD[48+rsi] + mov r15,QWORD[48+rsi] ; a[6] adcx r11,r13 - adox r12,rbp - adcx r12,rbp + adox r12,rbp ; of=0 + adcx r12,rbp ; cf=0 - mov QWORD[56+rdi],r8 - mov QWORD[64+rdi],r9 + mov QWORD[56+rdi],r8 ; t[7] + mov QWORD[64+rdi],r9 ; t[8] - mulx rax,r9,r14 - mov r8,QWORD[56+rsi] + mulx rax,r9,r14 ; a[5]*a[4] + mov r8,QWORD[56+rsi] ; a[7] adcx r9,r10 - mulx rbx,r10,r15 + mulx rbx,r10,r15 ; a[6]*a[4] adox r10,rax adcx r10,r11 - mulx rax,r11,r8 - mov rdx,r14 + mulx rax,r11,r8 ; a[7]*a[4] + mov rdx,r14 ; a[5] adox r11,rbx adcx r11,r12 +; adox %rbp,%rax # of=0 + adcx rax,rbp ; cf=0 - adcx rax,rbp - - mulx rbx,r14,r15 - mulx r13,r12,r8 - mov rdx,r15 + mulx rbx,r14,r15 ; a[6]*a[5] + mulx r13,r12,r8 ; a[7]*a[5] + mov rdx,r15 ; a[6] lea rsi,[64+rsi] adcx r11,r14 adox r12,rbx @@ -3014,19 +3014,19 @@ adox r13,rbp DB 0x67,0x67 - mulx r14,r8,r8 + mulx r14,r8,r8 ; a[7]*a[6] adcx r13,r8 adcx r14,rbp cmp rsi,QWORD[((8+8))+rsp] je NEAR $L$sqrx8x_outer_break - neg rcx + neg rcx ; mov %rcx,%cf mov rcx,-8 mov r15,rbp mov r8,QWORD[64+rdi] - adcx r9,QWORD[72+rdi] - adcx r10,QWORD[80+rdi] + adcx r9,QWORD[72+rdi] ; +=t[9] + adcx r10,QWORD[80+rdi] ; ... adcx r11,QWORD[88+rdi] adc r12,QWORD[96+rdi] adc r13,QWORD[104+rdi] @@ -3034,24 +3034,24 @@ adc r15,QWORD[120+rdi] lea rbp,[rsi] lea rdi,[128+rdi] - sbb rax,rax + sbb rax,rax ; mov %cf,%rcx - mov rdx,QWORD[((-64))+rsi] - mov QWORD[((16+8))+rsp],rax + mov rdx,QWORD[((-64))+rsi] ; a[0] + mov QWORD[((16+8))+rsp],rax ; offload %rcx mov QWORD[((24+8))+rsp],rdi - - xor eax,eax +; lea 8*8(%rdi),%rdi # see 2*8*8(%rdi) above + xor eax,eax ; cf=0, of=0 jmp NEAR $L$sqrx8x_loop ALIGN 32 $L$sqrx8x_loop: mov rbx,r8 - mulx r8,rax,QWORD[rbp] - adcx rbx,rax + mulx r8,rax,QWORD[rbp] ; a[8]*a[i] + adcx rbx,rax ; +=t[8] adox r8,r9 - mulx r9,rax,QWORD[8+rbp] + mulx r9,rax,QWORD[8+rbp] ; ... adcx r8,rax adox r9,r10 @@ -3063,7 +3063,7 @@ adcx r10,rax adox r11,r12 - DB 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 + DB 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 ; mulx 4*8(%rbp),%rax,%r12 adcx r11,rax adox r12,r13 @@ -3072,27 +3072,27 @@ adox r13,r14 mulx r14,rax,QWORD[48+rbp] - mov QWORD[rcx*8+rdi],rbx + mov QWORD[rcx*8+rdi],rbx ; store t[8+i] mov ebx,0 adcx r13,rax adox r14,r15 - DB 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 - mov rdx,QWORD[8+rcx*8+rsi] + DB 0xc4,0x62,0xfb,0xf6,0xbd,0x38,0x00,0x00,0x00 ; mulx 7*8(%rbp),%rax,%r15 + mov rdx,QWORD[8+rcx*8+rsi] ; a[i] adcx r14,rax - adox r15,rbx - adcx r15,rbx + adox r15,rbx ; %rbx is 0, of=0 + adcx r15,rbx ; cf=0 DB 0x67 - inc rcx + inc rcx ; of=0 jnz NEAR $L$sqrx8x_loop lea rbp,[64+rbp] mov rcx,-8 - cmp rbp,QWORD[((8+8))+rsp] + cmp rbp,QWORD[((8+8))+rsp] ; done? je NEAR $L$sqrx8x_break - sub rbx,QWORD[((16+8))+rsp] + sub rbx,QWORD[((16+8))+rsp] ; mov 16(%rsp),%cf DB 0x66 mov rdx,QWORD[((-64))+rsi] adcx r8,QWORD[rdi] @@ -3105,19 +3105,19 @@ adc r15,QWORD[56+rdi] lea rdi,[64+rdi] DB 0x67 - sbb rax,rax - xor ebx,ebx - mov QWORD[((16+8))+rsp],rax + sbb rax,rax ; mov %cf,%rax + xor ebx,ebx ; cf=0, of=0 + mov QWORD[((16+8))+rsp],rax ; offload carry jmp NEAR $L$sqrx8x_loop ALIGN 32 $L$sqrx8x_break: xor rbp,rbp - sub rbx,QWORD[((16+8))+rsp] + sub rbx,QWORD[((16+8))+rsp] ; mov 16(%rsp),%cf adcx r8,rbp - mov rcx,QWORD[((24+8))+rsp] + mov rcx,QWORD[((24+8))+rsp] ; initial %rdi, borrow %rcx adcx r9,rbp - mov rdx,QWORD[rsi] + mov rdx,QWORD[rsi] ; a[8], modulo-scheduled adc r10,0 mov QWORD[rdi],r8 adc r11,0 @@ -3125,7 +3125,7 @@ adc r13,0 adc r14,0 adc r15,0 - cmp rdi,rcx + cmp rdi,rcx ; cf=0, of=0 je NEAR $L$sqrx8x_outer_loop mov QWORD[8+rdi],r9 @@ -3147,57 +3147,57 @@ ALIGN 32 $L$sqrx8x_outer_break: - mov QWORD[72+rdi],r9 - movq rcx,xmm3 - mov QWORD[80+rdi],r10 + mov QWORD[72+rdi],r9 ; t[9] + movq rcx,xmm3 ; -%r9 + mov QWORD[80+rdi],r10 ; ... mov QWORD[88+rdi],r11 mov QWORD[96+rdi],r12 mov QWORD[104+rdi],r13 mov QWORD[112+rdi],r14 lea rdi,[((48+8))+rsp] - mov rdx,QWORD[rcx*1+rsi] + mov rdx,QWORD[rcx*1+rsi] ; a[0] - mov r11,QWORD[8+rdi] - xor r10,r10 - mov r9,QWORD[((0+8))+rsp] + mov r11,QWORD[8+rdi] ; t[1] + xor r10,r10 ; t[0], of=0, cf=0 + mov r9,QWORD[((0+8))+rsp] ; restore %r9 adox r11,r11 - mov r12,QWORD[16+rdi] - mov r13,QWORD[24+rdi] - + mov r12,QWORD[16+rdi] ; t[2] # prefetch + mov r13,QWORD[24+rdi] ; t[3] # prefetch +; jmp .Lsqrx4x_shift_n_add # happens to be aligned ALIGN 32 $L$sqrx4x_shift_n_add: mulx rbx,rax,rdx adox r12,r12 adcx rax,r10 - DB 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 - DB 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 + DB 0x48,0x8b,0x94,0x0e,0x08,0x00,0x00,0x00 ; mov 8(%rsi,%rcx),%rdx # a[i+1] # prefetch + DB 0x4c,0x8b,0x97,0x20,0x00,0x00,0x00 ; mov 32(%rdi),%r10 # t[2*i+4] # prefetch adox r13,r13 adcx rbx,r11 - mov r11,QWORD[40+rdi] + mov r11,QWORD[40+rdi] ; t[2*i+4+1] # prefetch mov QWORD[rdi],rax mov QWORD[8+rdi],rbx mulx rbx,rax,rdx adox r10,r10 adcx rax,r12 - mov rdx,QWORD[16+rcx*1+rsi] - mov r12,QWORD[48+rdi] + mov rdx,QWORD[16+rcx*1+rsi] ; a[i+2] # prefetch + mov r12,QWORD[48+rdi] ; t[2*i+6] # prefetch adox r11,r11 adcx rbx,r13 - mov r13,QWORD[56+rdi] + mov r13,QWORD[56+rdi] ; t[2*i+6+1] # prefetch mov QWORD[16+rdi],rax mov QWORD[24+rdi],rbx mulx rbx,rax,rdx adox r12,r12 adcx rax,r10 - mov rdx,QWORD[24+rcx*1+rsi] + mov rdx,QWORD[24+rcx*1+rsi] ; a[i+3] # prefetch lea rcx,[32+rcx] - mov r10,QWORD[64+rdi] + mov r10,QWORD[64+rdi] ; t[2*i+8] # prefetch adox r13,r13 adcx rbx,r11 - mov r11,QWORD[72+rdi] + mov r11,QWORD[72+rdi] ; t[2*i+8+1] # prefetch mov QWORD[32+rdi],rax mov QWORD[40+rdi],rbx @@ -3205,11 +3205,11 @@ adox r10,r10 adcx rax,r12 jrcxz $L$sqrx4x_shift_n_add_break - DB 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 + DB 0x48,0x8b,0x94,0x0e,0x00,0x00,0x00,0x00 ; mov 0(%rsi,%rcx),%rdx # a[i+4] # prefetch adox r11,r11 adcx rbx,r13 - mov r12,QWORD[80+rdi] - mov r13,QWORD[88+rdi] + mov r12,QWORD[80+rdi] ; t[2*i+10] # prefetch + mov r13,QWORD[88+rdi] ; t[2*i+10+1] # prefetch mov QWORD[48+rdi],rax mov QWORD[56+rdi],rbx lea rdi,[64+rdi] @@ -3221,18 +3221,18 @@ adcx rbx,r13 mov QWORD[48+rdi],rax mov QWORD[56+rdi],rbx - lea rdi,[64+rdi] + lea rdi,[64+rdi] ; end of t[] buffer movq rbp,xmm2 __bn_sqrx8x_reduction: - xor eax,eax - mov rbx,QWORD[((32+8))+rsp] - mov rdx,QWORD[((48+8))+rsp] - lea rcx,[((-64))+r9*1+rbp] + xor eax,eax ; initial top-most carry bit + mov rbx,QWORD[((32+8))+rsp] ; n0 + mov rdx,QWORD[((48+8))+rsp] ; "%r8", 8*0(%rdi) + lea rcx,[((-64))+r9*1+rbp] ; end of n[] +; lea 48+8(%rsp,%r9,2),%rdi # end of t[] buffer + mov QWORD[((0+8))+rsp],rcx ; save end of n[] + mov QWORD[((8+8))+rsp],rdi ; save end of t[] - mov QWORD[((0+8))+rsp],rcx - mov QWORD[((8+8))+rsp],rdi - - lea rdi,[((48+8))+rsp] + lea rdi,[((48+8))+rsp] ; initial t[] window jmp NEAR $L$sqrx8x_reduction_loop ALIGN 32 @@ -3242,25 +3242,25 @@ mov r11,QWORD[24+rdi] mov r12,QWORD[32+rdi] mov r8,rdx - imul rdx,rbx + imul rdx,rbx ; n0*a[i] mov r13,QWORD[40+rdi] mov r14,QWORD[48+rdi] mov r15,QWORD[56+rdi] - mov QWORD[((24+8))+rsp],rax + mov QWORD[((24+8))+rsp],rax ; store top-most carry bit lea rdi,[64+rdi] - xor rsi,rsi + xor rsi,rsi ; cf=0,of=0 mov rcx,-8 jmp NEAR $L$sqrx8x_reduce ALIGN 32 $L$sqrx8x_reduce: mov rbx,r8 - mulx r8,rax,QWORD[rbp] - adcx rax,rbx + mulx r8,rax,QWORD[rbp] ; n[0] + adcx rax,rbx ; discarded adox r8,r9 - mulx r9,rbx,QWORD[8+rbp] + mulx r9,rbx,QWORD[8+rbp] ; n[1] adcx r8,rbx adox r9,r10 @@ -3272,15 +3272,15 @@ adcx r10,rbx adox r11,r12 - DB 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 + DB 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 ; mulx 8*4(%rbp),%rbx,%r12 mov rax,rdx mov rdx,r8 adcx r11,rbx adox r12,r13 - mulx rdx,rbx,QWORD[((32+8))+rsp] + mulx rdx,rbx,QWORD[((32+8))+rsp] ; %rdx discarded mov rdx,rax - mov QWORD[((64+48+8))+rcx*8+rsp],rax + mov QWORD[((64+48+8))+rcx*8+rsp],rax ; put aside n0*a[i] mulx r13,rax,QWORD[40+rbp] adcx r12,rax @@ -3293,18 +3293,18 @@ mulx r15,rax,QWORD[56+rbp] mov rdx,rbx adcx r14,rax - adox r15,rsi - adcx r15,rsi + adox r15,rsi ; %rsi is 0 + adcx r15,rsi ; cf=0 DB 0x67,0x67,0x67 - inc rcx + inc rcx ; of=0 jnz NEAR $L$sqrx8x_reduce - mov rax,rsi - cmp rbp,QWORD[((0+8))+rsp] + mov rax,rsi ; xor %rax,%rax + cmp rbp,QWORD[((0+8))+rsp] ; end of n[]? jae NEAR $L$sqrx8x_no_tail - mov rdx,QWORD[((48+8))+rsp] + mov rdx,QWORD[((48+8))+rsp] ; pull n0*a[0] add r8,QWORD[rdi] lea rbp,[64+rbp] mov rcx,-8 @@ -3316,9 +3316,9 @@ adc r14,QWORD[48+rdi] adc r15,QWORD[56+rdi] lea rdi,[64+rdi] - sbb rax,rax + sbb rax,rax ; top carry - xor rsi,rsi + xor rsi,rsi ; of=0, cf=0 mov QWORD[((16+8))+rsp],rax jmp NEAR $L$sqrx8x_tail @@ -3341,7 +3341,7 @@ adcx r10,rax adox r11,r12 - DB 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 + DB 0xc4,0x62,0xfb,0xf6,0xa5,0x20,0x00,0x00,0x00 ; mulx 8*4(%rbp),%rax,%r12 adcx r11,rax adox r12,r13 @@ -3354,21 +3354,21 @@ adox r14,r15 mulx r15,rax,QWORD[56+rbp] - mov rdx,QWORD[((72+48+8))+rcx*8+rsp] + mov rdx,QWORD[((72+48+8))+rcx*8+rsp] ; pull n0*a[i] adcx r14,rax adox r15,rsi - mov QWORD[rcx*8+rdi],rbx + mov QWORD[rcx*8+rdi],rbx ; save result mov rbx,r8 - adcx r15,rsi + adcx r15,rsi ; cf=0 - inc rcx + inc rcx ; of=0 jnz NEAR $L$sqrx8x_tail - cmp rbp,QWORD[((0+8))+rsp] - jae NEAR $L$sqrx8x_tail_done + cmp rbp,QWORD[((0+8))+rsp] ; end of n[]? + jae NEAR $L$sqrx8x_tail_done ; break out of loop - sub rsi,QWORD[((16+8))+rsp] - mov rdx,QWORD[((48+8))+rsp] + sub rsi,QWORD[((16+8))+rsp] ; mov 16(%rsp),%cf + mov rdx,QWORD[((48+8))+rsp] ; pull n0*a[0] lea rbp,[64+rbp] adc r8,QWORD[rdi] adc r9,QWORD[8+rdi] @@ -3380,16 +3380,16 @@ adc r15,QWORD[56+rdi] lea rdi,[64+rdi] sbb rax,rax - sub rcx,8 + sub rcx,8 ; mov $-8,%rcx - xor rsi,rsi + xor rsi,rsi ; of=0, cf=0 mov QWORD[((16+8))+rsp],rax jmp NEAR $L$sqrx8x_tail ALIGN 32 $L$sqrx8x_tail_done: xor rax,rax - add r8,QWORD[((24+8))+rsp] + add r8,QWORD[((24+8))+rsp] ; can this overflow? adc r9,0 adc r10,0 adc r11,0 @@ -3399,26 +3399,26 @@ adc r15,0 adc rax,0 - sub rsi,QWORD[((16+8))+rsp] -$L$sqrx8x_no_tail: + sub rsi,QWORD[((16+8))+rsp] ; mov 16(%rsp),%cf +$L$sqrx8x_no_tail: ; %cf is 0 if jumped here adc r8,QWORD[rdi] movq rcx,xmm3 adc r9,QWORD[8+rdi] mov rsi,QWORD[56+rbp] - movq rbp,xmm2 + movq rbp,xmm2 ; restore %rbp adc r10,QWORD[16+rdi] adc r11,QWORD[24+rdi] adc r12,QWORD[32+rdi] adc r13,QWORD[40+rdi] adc r14,QWORD[48+rdi] adc r15,QWORD[56+rdi] - adc rax,0 + adc rax,0 ; top-most carry - mov rbx,QWORD[((32+8))+rsp] - mov rdx,QWORD[64+rcx*1+rdi] + mov rbx,QWORD[((32+8))+rsp] ; n0 + mov rdx,QWORD[64+rcx*1+rdi] ; modulo-scheduled "%r8" - mov QWORD[rdi],r8 - lea r8,[64+rdi] + mov QWORD[rdi],r8 ; store top 512 bits + lea r8,[64+rdi] ; borrow %r8 mov QWORD[8+rdi],r9 mov QWORD[16+rdi],r10 mov QWORD[24+rdi],r11 @@ -3427,8 +3427,8 @@ mov QWORD[48+rdi],r14 mov QWORD[56+rdi],r15 - lea rdi,[64+rcx*1+rdi] - cmp r8,QWORD[((8+8))+rsp] + lea rdi,[64+rcx*1+rdi] ; start of current t[] window + cmp r8,QWORD[((8+8))+rsp] ; end of t[]? jb NEAR $L$sqrx8x_reduction_loop ret @@ -3438,14 +3438,14 @@ __bn_postx4x_internal: mov r12,QWORD[rbp] - mov r10,rcx - mov r9,rcx + mov r10,rcx ; -%r9 + mov r9,rcx ; -%r9 neg rax sar rcx,3+2 - - movq rdx,xmm1 - movq rsi,xmm1 - dec r12 +; lea 48+8(%rsp,%r9),%rdi + movq rdx,xmm1 ; restore %rdx + movq rsi,xmm1 ; prepare for back-to-back call + dec r12 ; so that after 'not' we get -n[0] mov r13,QWORD[8+rbp] xor r8,r8 mov r14,QWORD[16+rbp] @@ -3465,7 +3465,7 @@ andn r14,r14,rax andn r15,r15,rax - neg r8 + neg r8 ; mov %r8,%cf adc r12,QWORD[rdi] adc r13,QWORD[8+rdi] adc r14,QWORD[16+rdi] @@ -3473,7 +3473,7 @@ mov QWORD[rdx],r12 lea rdi,[32+rdi] mov QWORD[8+rdx],r13 - sbb r8,r8 + sbb r8,r8 ; mov %cf,%r8 mov QWORD[16+rdx],r14 mov QWORD[24+rdx],r15 lea rdx,[32+rdx] @@ -3481,7 +3481,7 @@ inc rcx jnz NEAR $L$sqrx4x_sub - neg r9 + neg r9 ; restore %r9 ret @@ -3495,13 +3495,13 @@ cmp edx,0 jz NEAR $L$scatter_epilogue - - - - - - - +; %r8 stores 32 entries, t0 through t31. Each entry has %edx words. +; They are interleaved in memory as follows: +; +; t0[0] t1[0] t2[0] ... t31[0] +; t0[1] t1[1] t2[1] ... t31[1] +; ... +; t0[%edx-1] t1[%edx-1] t2[%edx-1] ... t31[%edx-1] lea r8,[r9*8+r8] $L$scatter: @@ -3521,97 +3521,97 @@ ALIGN 32 bn_gather5: -$L$SEH_begin_bn_gather5: +$L$SEH_begin_bn_gather5: ; Win64 thing, but harmless in other cases _CET_ENDBR +; I can't trust assembler to use specific encoding:-( + DB 0x4c,0x8d,0x14,0x24 ; lea (%rsp),%r10 - DB 0x4c,0x8d,0x14,0x24 - - DB 0x48,0x81,0xec,0x08,0x01,0x00,0x00 + DB 0x48,0x81,0xec,0x08,0x01,0x00,0x00 ; sub $0x108,%rsp lea rax,[$L$inc] - and rsp,-16 + and rsp,-16 ; shouldn't be formally required movd xmm5,r9d - movdqa xmm0,XMMWORD[rax] - movdqa xmm1,XMMWORD[16+rax] - lea r11,[128+r8] - lea rax,[128+rsp] + movdqa xmm0,XMMWORD[rax] ; 00000001000000010000000000000000 + movdqa xmm1,XMMWORD[16+rax] ; 00000002000000020000000200000002 + lea r11,[128+r8] ; size optimization + lea rax,[128+rsp] ; size optimization - pshufd xmm5,xmm5,0 + pshufd xmm5,xmm5,0 ; broadcast %r9d movdqa xmm4,xmm1 movdqa xmm2,xmm1 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[(-128)+rax],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[(-112)+rax],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[(-96)+rax],xmm2 movdqa xmm2,xmm4 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 movdqa XMMWORD[(-80)+rax],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[(-64)+rax],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[(-48)+rax],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[(-32)+rax],xmm2 movdqa xmm2,xmm4 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 movdqa XMMWORD[(-16)+rax],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[rax],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[16+rax],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[32+rax],xmm2 movdqa xmm2,xmm4 paddd xmm1,xmm0 - pcmpeqd xmm0,xmm5 + pcmpeqd xmm0,xmm5 ; compare to 1,0 movdqa XMMWORD[48+rax],xmm3 movdqa xmm3,xmm4 paddd xmm2,xmm1 - pcmpeqd xmm1,xmm5 + pcmpeqd xmm1,xmm5 ; compare to 3,2 movdqa XMMWORD[64+rax],xmm0 movdqa xmm0,xmm4 paddd xmm3,xmm2 - pcmpeqd xmm2,xmm5 + pcmpeqd xmm2,xmm5 ; compare to 5,4 movdqa XMMWORD[80+rax],xmm1 movdqa xmm1,xmm4 paddd xmm0,xmm3 - pcmpeqd xmm3,xmm5 + pcmpeqd xmm3,xmm5 ; compare to 7,6 movdqa XMMWORD[96+rax],xmm2 movdqa xmm2,xmm4 movdqa XMMWORD[112+rax],xmm3 @@ -3671,10 +3671,10 @@ por xmm5,xmm3 por xmm4,xmm5 lea r11,[256+r11] - - pshufd xmm0,xmm4,0x4e +; Combine the upper and lower halves of %xmm0. + pshufd xmm0,xmm4,0x4e ; Swap upper and lower halves. por xmm0,xmm4 - movq QWORD[rcx],xmm0 + movq QWORD[rcx],xmm0 ; m0=bp[0] lea rcx,[8+rcx] sub edx,1 jnz NEAR $L$gather @@ -3715,40 +3715,40 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; end of prologue label + cmp rbx,r10 ; context->Rip<end of prologue label jb NEAR $L$common_seh_tail - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; beginning of body label + cmp rbx,r10 ; context->Rip<body label jb NEAR $L$common_pop_regs - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - mov r10d,DWORD[8+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[8+r11] ; HandlerData[2] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail lea r10,[$L$mul_epilogue] cmp rbx,r10 ja NEAR $L$body_40 - mov r10,QWORD[192+r8] - mov rax,QWORD[8+r10*8+rax] + mov r10,QWORD[192+r8] ; pull %r9 + mov rax,QWORD[8+r10*8+rax] ; pull saved stack pointer jmp NEAR $L$common_pop_regs $L$body_40: - mov rax,QWORD[40+rax] + mov rax,QWORD[40+rax] ; pull saved stack pointer $L$common_pop_regs: mov rbx,QWORD[((-8))+rax] mov rbp,QWORD[((-16))+rax] @@ -3756,40 +3756,40 @@ mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R15 $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -3832,32 +3832,32 @@ $L$SEH_info_bn_mul_mont_gather5_nohw: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$mul_body wrt ..imagebase,$L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase + DD $L$mul_body wrt ..imagebase,$L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 $L$SEH_info_bn_mul4x_mont_gather5: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$mul4x_prologue wrt ..imagebase,$L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase + DD $L$mul4x_prologue wrt ..imagebase,$L$mul4x_body wrt ..imagebase,$L$mul4x_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 $L$SEH_info_bn_power5_nohw: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$power5_prologue wrt ..imagebase,$L$power5_body wrt ..imagebase,$L$power5_epilogue wrt ..imagebase + DD $L$power5_prologue wrt ..imagebase,$L$power5_body wrt ..imagebase,$L$power5_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 $L$SEH_info_bn_mulx4x_mont_gather5: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$mulx4x_prologue wrt ..imagebase,$L$mulx4x_body wrt ..imagebase,$L$mulx4x_epilogue wrt ..imagebase + DD $L$mulx4x_prologue wrt ..imagebase,$L$mulx4x_body wrt ..imagebase,$L$mulx4x_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 $L$SEH_info_bn_powerx5: DB 9,0,0,0 DD mul_handler wrt ..imagebase - DD $L$powerx5_prologue wrt ..imagebase,$L$powerx5_body wrt ..imagebase,$L$powerx5_epilogue wrt ..imagebase + DD $L$powerx5_prologue wrt ..imagebase,$L$powerx5_body wrt ..imagebase,$L$powerx5_epilogue wrt ..imagebase ; HandlerData[] ALIGN 8 $L$SEH_info_bn_gather5: DB 0x01,0x0b,0x03,0x0a - DB 0x0b,0x01,0x21,0x00 - DB 0x04,0xa3,0x00,0x00 + DB 0x0b,0x01,0x21,0x00 ; sub rsp,0x108 + DB 0x04,0xa3,0x00,0x00 ; lea r10,(rsp) ALIGN 8 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
diff --git a/gen/crypto/aes128gcmsiv-x86_64-apple.S b/gen/crypto/aes128gcmsiv-x86_64-apple.S index c3fcf6d..eff851d 100644 --- a/gen/crypto/aes128gcmsiv-x86_64-apple.S +++ b/gen/crypto/aes128gcmsiv-x86_64-apple.S
@@ -89,21 +89,21 @@ _CET_ENDBR vmovdqa (%rsi),%xmm0 vmovdqa %xmm0,%xmm1 - vmovdqa %xmm0,(%rdi) + vmovdqa %xmm0,(%rdi) // H call GFMUL - vmovdqa %xmm0,16(%rdi) + vmovdqa %xmm0,16(%rdi) // H^2 call GFMUL - vmovdqa %xmm0,32(%rdi) + vmovdqa %xmm0,32(%rdi) // H^3 call GFMUL - vmovdqa %xmm0,48(%rdi) + vmovdqa %xmm0,48(%rdi) // H^4 call GFMUL - vmovdqa %xmm0,64(%rdi) + vmovdqa %xmm0,64(%rdi) // H^5 call GFMUL - vmovdqa %xmm0,80(%rdi) + vmovdqa %xmm0,80(%rdi) // H^6 call GFMUL - vmovdqa %xmm0,96(%rdi) + vmovdqa %xmm0,96(%rdi) // H^7 call GFMUL - vmovdqa %xmm0,112(%rdi) + vmovdqa %xmm0,112(%rdi) // H^8 ret @@ -117,17 +117,17 @@ _CET_ENDBR vmovdqa (%rsi),%xmm0 vmovdqa %xmm0,%xmm1 - vmovdqa %xmm0,(%rdi) + vmovdqa %xmm0,(%rdi) // H call GFMUL - vmovdqa %xmm0,16(%rdi) + vmovdqa %xmm0,16(%rdi) // H^2 call GFMUL - vmovdqa %xmm0,32(%rdi) + vmovdqa %xmm0,32(%rdi) // H^3 call GFMUL - vmovdqa %xmm0,48(%rdi) + vmovdqa %xmm0,48(%rdi) // H^4 call GFMUL - vmovdqa %xmm0,64(%rdi) + vmovdqa %xmm0,64(%rdi) // H^5 call GFMUL - vmovdqa %xmm0,80(%rdi) + vmovdqa %xmm0,80(%rdi) // H^6 ret @@ -146,8 +146,8 @@ L$htable_polyval_start: vzeroall - - +// We hash 8 blocks each iteration. If the total number of blocks is not a +// multiple of 8, we first hash the leading n%8 blocks. movq %rdx,%r11 andq $127,%r11 @@ -159,7 +159,7 @@ subq $16,%r11 - +// hash first prefix block vmovdqu (%rsi),%xmm0 vpxor %xmm1,%xmm0,%xmm0 @@ -174,12 +174,12 @@ jnz L$htable_polyval_prefix_loop jmp L$htable_polyval_prefix_complete - +// hash remaining prefix blocks (up to 7 total prefix blocks) .p2align 6 L$htable_polyval_prefix_loop: subq $16,%r11 - vmovdqu (%rsi),%xmm0 + vmovdqu (%rsi),%xmm0 // next data block vpclmulqdq $0x00,(%rdi,%r11,1),%xmm0,%xmm6 vpxor %xmm6,%xmm3,%xmm3 @@ -206,10 +206,10 @@ jmp L$htable_polyval_main_loop L$htable_polyval_no_prefix: - - - - +// At this point we know the number of blocks is a multiple of 8. However, +// the reduction in the main loop includes a multiplication by x^(-128). In +// order to counter this, the existing tag needs to be multiplied by x^128. +// In practice, this just means that it is loaded into %xmm9, not %xmm1. vpxor %xmm1,%xmm1,%xmm1 vmovdqa (%rcx),%xmm9 @@ -218,7 +218,7 @@ subq $0x80,%rdx jb L$htable_polyval_out - vmovdqu 112(%rsi),%xmm0 + vmovdqu 112(%rsi),%xmm0 // Ii vpclmulqdq $0x01,(%rdi),%xmm0,%xmm5 vpclmulqdq $0x00,(%rdi),%xmm0,%xmm3 @@ -226,7 +226,7 @@ vpclmulqdq $0x10,(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 96(%rsi),%xmm0 vpclmulqdq $0x01,16(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 @@ -238,10 +238,10 @@ vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 80(%rsi),%xmm0 - vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 + vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 // reduction stage 1a vpalignr $8,%xmm1,%xmm1,%xmm1 vpclmulqdq $0x01,32(%rdi),%xmm0,%xmm6 @@ -254,8 +254,8 @@ vpxor %xmm6,%xmm5,%xmm5 - vpxor %xmm7,%xmm1,%xmm1 - + vpxor %xmm7,%xmm1,%xmm1 // reduction stage 1b +// ######################################################## vmovdqu 64(%rsi),%xmm0 vpclmulqdq $0x01,48(%rdi),%xmm0,%xmm6 @@ -267,10 +267,10 @@ vpclmulqdq $0x10,48(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 48(%rsi),%xmm0 - vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 + vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 // reduction stage 2a vpalignr $8,%xmm1,%xmm1,%xmm1 vpclmulqdq $0x01,64(%rdi),%xmm0,%xmm6 @@ -283,8 +283,8 @@ vpxor %xmm6,%xmm5,%xmm5 - vpxor %xmm7,%xmm1,%xmm1 - + vpxor %xmm7,%xmm1,%xmm1 // reduction stage 2b +// ######################################################## vmovdqu 32(%rsi),%xmm0 vpclmulqdq $0x01,80(%rdi),%xmm0,%xmm6 @@ -297,8 +297,8 @@ vpxor %xmm6,%xmm5,%xmm5 - vpxor %xmm9,%xmm1,%xmm1 - + vpxor %xmm9,%xmm1,%xmm1 // reduction finalize +// ######################################################## vmovdqu 16(%rsi),%xmm0 vpclmulqdq $0x01,96(%rdi),%xmm0,%xmm6 @@ -310,7 +310,7 @@ vpclmulqdq $0x10,96(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 0(%rsi),%xmm0 vpxor %xmm1,%xmm0,%xmm0 @@ -323,7 +323,7 @@ vpclmulqdq $0x10,112(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vpsrldq $8,%xmm5,%xmm6 vpslldq $8,%xmm5,%xmm5 @@ -333,7 +333,7 @@ leaq 128(%rsi),%rsi jmp L$htable_polyval_main_loop - +// ######################################################## L$htable_polyval_out: vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm6 @@ -363,24 +363,24 @@ ret L$polyval_horner_start: - - +// We will start with L GFMULS for POLYVAL(BIG_BUFFER) +// RES = GFMUL(RES, H) xorq %r10,%r10 - shlq $4,%rcx + shlq $4,%rcx // L contains number of bytes to process vmovdqa (%rsi),%xmm1 vmovdqa (%rdi),%xmm0 L$polyval_horner_loop: - vpxor (%rdx,%r10,1),%xmm0,%xmm0 - call GFMUL + vpxor (%rdx,%r10,1),%xmm0,%xmm0 // RES = RES + Xi + call GFMUL // RES = RES * H addq $16,%r10 cmpq %r10,%rcx jne L$polyval_horner_loop - +// calculation of T is complete. RES=T vmovdqa %xmm0,(%rdi) ret @@ -393,8 +393,8 @@ _aes128gcmsiv_aes_ks: _CET_ENDBR - vmovdqu (%rdi),%xmm1 - vmovdqa %xmm1,(%rsi) + vmovdqu (%rdi),%xmm1 // xmm1 = user key + vmovdqa %xmm1,(%rsi) // rsi points to output vmovdqa con1(%rip),%xmm0 vmovdqa mask(%rip),%xmm15 @@ -402,9 +402,9 @@ movq $8,%rax L$ks128_loop: - addq $16,%rsi + addq $16,%rsi // rsi points for next key subq $1,%rax - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // xmm2 = shuffled user key vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 vpslldq $4,%xmm1,%xmm3 @@ -500,19 +500,19 @@ _aes128gcmsiv_aes_ks_enc_x1: _CET_ENDBR - vmovdqa (%rcx),%xmm1 + vmovdqa (%rcx),%xmm1 // xmm1 = first 16 bytes of random key vmovdqa 0(%rdi),%xmm4 - vmovdqa %xmm1,(%rdx) + vmovdqa %xmm1,(%rdx) // KEY[0] = first 16 bytes of random key vpxor %xmm1,%xmm4,%xmm4 - vmovdqa con1(%rip),%xmm0 - vmovdqa mask(%rip),%xmm15 + vmovdqa con1(%rip),%xmm0 // xmm0 = 1,1,1,1 + vmovdqa mask(%rip),%xmm15 // xmm15 = mask - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -521,10 +521,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,16(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -533,10 +533,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,32(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -545,10 +545,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,48(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -557,10 +557,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,64(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -569,10 +569,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,80(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -581,10 +581,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,96(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -593,10 +593,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,112(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -608,10 +608,10 @@ vmovdqa con2(%rip),%xmm0 - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -620,9 +620,9 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,144(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -644,11 +644,11 @@ _aes128gcmsiv_kdf: _CET_ENDBR +// parameter 1: %rdi Pointer to NONCE +// parameter 2: %rsi Pointer to CT +// parameter 4: %rdx Pointer to keys - - - - vmovdqa (%rdx),%xmm1 + vmovdqa (%rdx),%xmm1 // xmm1 = first 16 bytes of random key vmovdqa 0(%rdi),%xmm9 vmovdqa and_mask(%rip),%xmm12 vmovdqa one(%rip),%xmm13 @@ -749,20 +749,20 @@ pushq %r13 - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks movq %r8,%r10 shlq $62,%r10 shrq $62,%r10 - +// make IV from TAG vmovdqa (%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // IV = [1]TAG[126...32][00..00] - vmovdqu four(%rip),%xmm4 - vmovdqa %xmm15,%xmm0 - vpaddd one(%rip),%xmm15,%xmm1 - vpaddd two(%rip),%xmm15,%xmm2 - vpaddd three(%rip),%xmm15,%xmm3 + vmovdqu four(%rip),%xmm4 // Register to increment counters + vmovdqa %xmm15,%xmm0 // CTR1 = TAG[1][127...32][00..00] + vpaddd one(%rip),%xmm15,%xmm1 // CTR2 = TAG[1][127...32][00..01] + vpaddd two(%rip),%xmm15,%xmm2 // CTR3 = TAG[1][127...32][00..02] + vpaddd three(%rip),%xmm15,%xmm3 // CTR4 = TAG[1][127...32][00..03] shrq $2,%r8 je L$128_enc_msg_x4_check_remainder @@ -850,7 +850,7 @@ vaesenclast %xmm12,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm5,%xmm5 vpxor 16(%rdi),%xmm6,%xmm6 vpxor 32(%rdi),%xmm7,%xmm7 @@ -873,10 +873,10 @@ je L$128_enc_msg_x4_out L$128_enc_msg_x4_loop2: - - +// enc each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm5 - vpaddd one(%rip),%xmm0,%xmm0 + vpaddd one(%rip),%xmm0,%xmm0 // inc counter vpxor (%rcx),%xmm5,%xmm5 vaesenc 16(%rcx),%xmm5,%xmm5 @@ -890,7 +890,7 @@ vaesenc 144(%rcx),%xmm5,%xmm5 vaesenclast 160(%rcx),%xmm5,%xmm5 - +// XOR with plaintext vpxor (%rdi),%xmm5,%xmm5 vmovdqu %xmm5,(%rsi) @@ -930,29 +930,29 @@ movq %rsp,%rbp - +// Place in stack subq $128,%rsp andq $-64,%rsp - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks movq %r8,%r10 shlq $61,%r10 shrq $61,%r10 - +// make IV from TAG vmovdqu (%rdx),%xmm1 - vpor OR_MASK(%rip),%xmm1,%xmm1 + vpor OR_MASK(%rip),%xmm1,%xmm1 // TMP1= IV = [1]TAG[126...32][00..00] - +// store counter8 in the stack vpaddd seven(%rip),%xmm1,%xmm0 - vmovdqu %xmm0,(%rsp) - vpaddd one(%rip),%xmm1,%xmm9 - vpaddd two(%rip),%xmm1,%xmm10 - vpaddd three(%rip),%xmm1,%xmm11 - vpaddd four(%rip),%xmm1,%xmm12 - vpaddd five(%rip),%xmm1,%xmm13 - vpaddd six(%rip),%xmm1,%xmm14 - vmovdqa %xmm1,%xmm0 + vmovdqu %xmm0,(%rsp) // CTR8 = TAG[127...32][00..07] + vpaddd one(%rip),%xmm1,%xmm9 // CTR2 = TAG[127...32][00..01] + vpaddd two(%rip),%xmm1,%xmm10 // CTR3 = TAG[127...32][00..02] + vpaddd three(%rip),%xmm1,%xmm11 // CTR4 = TAG[127...32][00..03] + vpaddd four(%rip),%xmm1,%xmm12 // CTR5 = TAG[127...32][00..04] + vpaddd five(%rip),%xmm1,%xmm13 // CTR6 = TAG[127...32][00..05] + vpaddd six(%rip),%xmm1,%xmm14 // CTR7 = TAG[127...32][00..06] + vmovdqa %xmm1,%xmm0 // CTR1 = TAG[127...32][00..00] shrq $3,%r8 je L$128_enc_msg_x8_check_remainder @@ -971,7 +971,7 @@ vmovdqa %xmm12,%xmm5 vmovdqa %xmm13,%xmm6 vmovdqa %xmm14,%xmm7 - +// move from stack vmovdqu (%rsp),%xmm8 vpxor (%rcx),%xmm1,%xmm1 @@ -993,7 +993,7 @@ vaesenc %xmm15,%xmm7,%xmm7 vaesenc %xmm15,%xmm8,%xmm8 - vmovdqu (%rsp),%xmm14 + vmovdqu (%rsp),%xmm14 // deal with CTR8 vpaddd eight(%rip),%xmm14,%xmm14 vmovdqu %xmm14,(%rsp) vmovdqu 32(%rcx),%xmm15 @@ -1094,7 +1094,7 @@ vaesenclast %xmm15,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm1,%xmm1 vpxor 16(%rdi),%xmm2,%xmm2 vpxor 32(%rdi),%xmm3,%xmm3 @@ -1125,10 +1125,10 @@ je L$128_enc_msg_x8_out L$128_enc_msg_x8_loop2: - - +// enc each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm1 - vpaddd one(%rip),%xmm0,%xmm0 + vpaddd one(%rip),%xmm0,%xmm0 // inc counter vpxor (%rcx),%xmm1,%xmm1 vaesenc 16(%rcx),%xmm1,%xmm1 @@ -1142,7 +1142,7 @@ vaesenc 144(%rcx),%xmm1,%xmm1 vaesenclast 160(%rcx),%xmm1,%xmm1 - +// XOR with Plaintext vpxor (%rdi),%xmm1,%xmm1 vmovdqu %xmm1,(%rsi) @@ -1180,10 +1180,10 @@ L$128_dec_start: vzeroupper vmovdqa (%rdx),%xmm0 - - +// The claimed tag is provided after the current calculated tag value. +// CTRBLKs is made from it. vmovdqu 16(%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // CTR = [1]TAG[126...32][00..00] movq %rdx,%rax leaq 32(%rax),%rax @@ -1191,11 +1191,11 @@ andq $~15,%r9 - +// If less then 6 blocks, make singles cmpq $96,%r9 jb L$128_dec_loop2 - +// Decrypt the first six blocks subq $96,%r9 vmovdqa %xmm15,%xmm7 vpaddd one(%rip),%xmm7,%xmm8 @@ -1292,7 +1292,7 @@ vaesenclast %xmm4,%xmm11,%xmm11 vaesenclast %xmm4,%xmm12,%xmm12 - +// XOR with CT vpxor 0(%rdi),%xmm7,%xmm7 vpxor 16(%rdi),%xmm8,%xmm8 vpxor 32(%rdi),%xmm9,%xmm9 @@ -1311,7 +1311,7 @@ addq $96,%rsi jmp L$128_dec_loop1 - +// Decrypt 6 blocks each time while hashing previous 6 blocks .p2align 6 L$128_dec_loop1: cmpq $96,%r9 @@ -1623,9 +1623,9 @@ vpxor %xmm5,%xmm0,%xmm0 L$128_dec_loop2: +// Here we encrypt any remaining whole block - - +// if there are no whole blocks cmpq $16,%r9 jb L$128_dec_out subq $16,%r9 @@ -1680,7 +1680,7 @@ vaesenc 112(%rdx),%xmm1,%xmm1 vaesenc 128(%rdx),%xmm1,%xmm1 vaesenc 144(%rdx),%xmm1,%xmm1 - vaesenclast 160(%rdx),%xmm1,%xmm1 + vaesenclast 160(%rdx),%xmm1,%xmm1 // STATE_1 == IV vmovdqa %xmm1,(%rsi) @@ -1695,14 +1695,14 @@ _aes256gcmsiv_aes_ks_enc_x1: _CET_ENDBR - vmovdqa con1(%rip),%xmm0 - vmovdqa mask(%rip),%xmm15 + vmovdqa con1(%rip),%xmm0 // CON_MASK = 1,1,1,1 + vmovdqa mask(%rip),%xmm15 // MASK_256 vmovdqa (%rdi),%xmm8 - vmovdqa (%rcx),%xmm1 + vmovdqa (%rcx),%xmm1 // KEY_1 || KEY_2 [0..7] = user key vmovdqa 16(%rcx),%xmm3 vpxor %xmm1,%xmm8,%xmm8 vaesenc %xmm3,%xmm8,%xmm8 - vmovdqu %xmm1,(%rdx) + vmovdqu %xmm1,(%rdx) // First round key vmovdqu %xmm3,16(%rdx) vpxor %xmm14,%xmm14,%xmm14 @@ -1895,7 +1895,7 @@ vaesenc 176(%rdx),%xmm1,%xmm1 vaesenc 192(%rdx),%xmm1,%xmm1 vaesenc 208(%rdx),%xmm1,%xmm1 - vaesenclast 224(%rdx),%xmm1,%xmm1 + vaesenclast 224(%rdx),%xmm1,%xmm1 // %xmm1 == IV vmovdqa %xmm1,(%rsi) ret @@ -1914,7 +1914,7 @@ L$256_enc_msg_x4_start: movq %r8,%r10 - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks shlq $60,%r10 jz L$256_enc_msg_x4_start2 addq $1,%r8 @@ -1924,15 +1924,15 @@ shlq $62,%r10 shrq $62,%r10 - +// make IV from TAG vmovdqa (%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // IV = [1]TAG[126...32][00..00] - vmovdqa four(%rip),%xmm4 - vmovdqa %xmm15,%xmm0 - vpaddd one(%rip),%xmm15,%xmm1 - vpaddd two(%rip),%xmm15,%xmm2 - vpaddd three(%rip),%xmm15,%xmm3 + vmovdqa four(%rip),%xmm4 // Register to increment counters + vmovdqa %xmm15,%xmm0 // CTR1 = TAG[1][127...32][00..00] + vpaddd one(%rip),%xmm15,%xmm1 // CTR2 = TAG[1][127...32][00..01] + vpaddd two(%rip),%xmm15,%xmm2 // CTR3 = TAG[1][127...32][00..02] + vpaddd three(%rip),%xmm15,%xmm3 // CTR4 = TAG[1][127...32][00..03] shrq $2,%r8 je L$256_enc_msg_x4_check_remainder @@ -2044,7 +2044,7 @@ vaesenclast %xmm12,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm5,%xmm5 vpxor 16(%rdi),%xmm6,%xmm6 vpxor 32(%rdi),%xmm7,%xmm7 @@ -2067,11 +2067,11 @@ je L$256_enc_msg_x4_out L$256_enc_msg_x4_loop2: - - +// encrypt each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm5 - vpaddd one(%rip),%xmm0,%xmm0 + vpaddd one(%rip),%xmm0,%xmm0 // inc counter vpxor (%rcx),%xmm5,%xmm5 vaesenc 16(%rcx),%xmm5,%xmm5 vaesenc 32(%rcx),%xmm5,%xmm5 @@ -2088,7 +2088,7 @@ vaesenc 208(%rcx),%xmm5,%xmm5 vaesenclast 224(%rcx),%xmm5,%xmm5 - +// XOR with Plaintext vpxor (%rdi),%xmm5,%xmm5 vmovdqu %xmm5,(%rsi) @@ -2116,13 +2116,13 @@ ret L$256_enc_msg_x8_start: - +// Place in stack movq %rsp,%r11 subq $16,%r11 andq $-64,%r11 movq %r8,%r10 - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks shlq $60,%r10 jz L$256_enc_msg_x8_start2 addq $1,%r8 @@ -2132,20 +2132,20 @@ shlq $61,%r10 shrq $61,%r10 - +// Make IV from TAG vmovdqa (%rdx),%xmm1 - vpor OR_MASK(%rip),%xmm1,%xmm1 + vpor OR_MASK(%rip),%xmm1,%xmm1 // TMP1= IV = [1]TAG[126...32][00..00] - +// store counter8 on the stack vpaddd seven(%rip),%xmm1,%xmm0 - vmovdqa %xmm0,(%r11) - vpaddd one(%rip),%xmm1,%xmm9 - vpaddd two(%rip),%xmm1,%xmm10 - vpaddd three(%rip),%xmm1,%xmm11 - vpaddd four(%rip),%xmm1,%xmm12 - vpaddd five(%rip),%xmm1,%xmm13 - vpaddd six(%rip),%xmm1,%xmm14 - vmovdqa %xmm1,%xmm0 + vmovdqa %xmm0,(%r11) // CTR8 = TAG[127...32][00..07] + vpaddd one(%rip),%xmm1,%xmm9 // CTR2 = TAG[127...32][00..01] + vpaddd two(%rip),%xmm1,%xmm10 // CTR3 = TAG[127...32][00..02] + vpaddd three(%rip),%xmm1,%xmm11 // CTR4 = TAG[127...32][00..03] + vpaddd four(%rip),%xmm1,%xmm12 // CTR5 = TAG[127...32][00..04] + vpaddd five(%rip),%xmm1,%xmm13 // CTR6 = TAG[127...32][00..05] + vpaddd six(%rip),%xmm1,%xmm14 // CTR7 = TAG[127...32][00..06] + vmovdqa %xmm1,%xmm0 // CTR1 = TAG[127...32][00..00] shrq $3,%r8 jz L$256_enc_msg_x8_check_remainder @@ -2164,7 +2164,7 @@ vmovdqa %xmm12,%xmm5 vmovdqa %xmm13,%xmm6 vmovdqa %xmm14,%xmm7 - +// move from stack vmovdqa (%r11),%xmm8 vpxor (%rcx),%xmm1,%xmm1 @@ -2186,7 +2186,7 @@ vaesenc %xmm15,%xmm7,%xmm7 vaesenc %xmm15,%xmm8,%xmm8 - vmovdqa (%r11),%xmm14 + vmovdqa (%r11),%xmm14 // deal with CTR8 vpaddd eight(%rip),%xmm14,%xmm14 vmovdqa %xmm14,(%r11) vmovdqu 32(%rcx),%xmm15 @@ -2327,7 +2327,7 @@ vaesenclast %xmm15,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm1,%xmm1 vpxor 16(%rdi),%xmm2,%xmm2 vpxor 32(%rdi),%xmm3,%xmm3 @@ -2358,8 +2358,8 @@ je L$256_enc_msg_x8_out L$256_enc_msg_x8_loop2: - - +// encrypt each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm1 vpaddd one(%rip),%xmm0,%xmm0 @@ -2379,7 +2379,7 @@ vaesenc 208(%rcx),%xmm1,%xmm1 vaesenclast 224(%rcx),%xmm1,%xmm1 - +// XOR with Plaintext vpxor (%rdi),%xmm1,%xmm1 vmovdqu %xmm1,(%rsi) @@ -2409,10 +2409,10 @@ L$256_dec_start: vzeroupper vmovdqa (%rdx),%xmm0 - - +// The claimed tag is provided after the current calculated tag value. +// CTRBLKs is made from it. vmovdqu 16(%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // CTR = [1]TAG[126...32][00..00] movq %rdx,%rax leaq 32(%rax),%rax @@ -2420,11 +2420,11 @@ andq $~15,%r9 - +// If less then 6 blocks, make singles cmpq $96,%r9 jb L$256_dec_loop2 - +// Decrypt the first six blocks subq $96,%r9 vmovdqa %xmm15,%xmm7 vpaddd one(%rip),%xmm7,%xmm8 @@ -2553,7 +2553,7 @@ vaesenclast %xmm4,%xmm11,%xmm11 vaesenclast %xmm4,%xmm12,%xmm12 - +// XOR with CT vpxor 0(%rdi),%xmm7,%xmm7 vpxor 16(%rdi),%xmm8,%xmm8 vpxor 32(%rdi),%xmm9,%xmm9 @@ -2572,7 +2572,7 @@ addq $96,%rsi jmp L$256_dec_loop1 - +// Decrypt 6 blocks each time while hashing previous 6 blocks .p2align 6 L$256_dec_loop1: cmpq $96,%r9 @@ -2916,9 +2916,9 @@ vpxor %xmm5,%xmm0,%xmm0 L$256_dec_loop2: +// Here we encrypt any remaining whole block - - +// if there are no whole blocks cmpq $16,%r9 jb L$256_dec_out subq $16,%r9 @@ -2965,11 +2965,11 @@ _aes256gcmsiv_kdf: _CET_ENDBR +// parameter 1: %rdi Pointer to NONCE +// parameter 2: %rsi Pointer to CT +// parameter 4: %rdx Pointer to keys - - - - vmovdqa (%rdx),%xmm1 + vmovdqa (%rdx),%xmm1 // xmm1 = first 16 bytes of random key vmovdqa 0(%rdi),%xmm4 vmovdqa and_mask(%rip),%xmm11 vmovdqa one(%rip),%xmm8
diff --git a/gen/crypto/aes128gcmsiv-x86_64-linux.S b/gen/crypto/aes128gcmsiv-x86_64-linux.S index a8de4a9..a0e3040 100644 --- a/gen/crypto/aes128gcmsiv-x86_64-linux.S +++ b/gen/crypto/aes128gcmsiv-x86_64-linux.S
@@ -74,21 +74,21 @@ _CET_ENDBR vmovdqa (%rsi),%xmm0 vmovdqa %xmm0,%xmm1 - vmovdqa %xmm0,(%rdi) + vmovdqa %xmm0,(%rdi) // H call GFMUL - vmovdqa %xmm0,16(%rdi) + vmovdqa %xmm0,16(%rdi) // H^2 call GFMUL - vmovdqa %xmm0,32(%rdi) + vmovdqa %xmm0,32(%rdi) // H^3 call GFMUL - vmovdqa %xmm0,48(%rdi) + vmovdqa %xmm0,48(%rdi) // H^4 call GFMUL - vmovdqa %xmm0,64(%rdi) + vmovdqa %xmm0,64(%rdi) // H^5 call GFMUL - vmovdqa %xmm0,80(%rdi) + vmovdqa %xmm0,80(%rdi) // H^6 call GFMUL - vmovdqa %xmm0,96(%rdi) + vmovdqa %xmm0,96(%rdi) // H^7 call GFMUL - vmovdqa %xmm0,112(%rdi) + vmovdqa %xmm0,112(%rdi) // H^8 ret .cfi_endproc .size aesgcmsiv_htable_init, .-aesgcmsiv_htable_init @@ -101,17 +101,17 @@ _CET_ENDBR vmovdqa (%rsi),%xmm0 vmovdqa %xmm0,%xmm1 - vmovdqa %xmm0,(%rdi) + vmovdqa %xmm0,(%rdi) // H call GFMUL - vmovdqa %xmm0,16(%rdi) + vmovdqa %xmm0,16(%rdi) // H^2 call GFMUL - vmovdqa %xmm0,32(%rdi) + vmovdqa %xmm0,32(%rdi) // H^3 call GFMUL - vmovdqa %xmm0,48(%rdi) + vmovdqa %xmm0,48(%rdi) // H^4 call GFMUL - vmovdqa %xmm0,64(%rdi) + vmovdqa %xmm0,64(%rdi) // H^5 call GFMUL - vmovdqa %xmm0,80(%rdi) + vmovdqa %xmm0,80(%rdi) // H^6 ret .cfi_endproc .size aesgcmsiv_htable6_init, .-aesgcmsiv_htable6_init @@ -129,8 +129,8 @@ .Lhtable_polyval_start: vzeroall - - +// We hash 8 blocks each iteration. If the total number of blocks is not a +// multiple of 8, we first hash the leading n%8 blocks. movq %rdx,%r11 andq $127,%r11 @@ -142,7 +142,7 @@ subq $16,%r11 - +// hash first prefix block vmovdqu (%rsi),%xmm0 vpxor %xmm1,%xmm0,%xmm0 @@ -157,12 +157,12 @@ jnz .Lhtable_polyval_prefix_loop jmp .Lhtable_polyval_prefix_complete - +// hash remaining prefix blocks (up to 7 total prefix blocks) .align 64 .Lhtable_polyval_prefix_loop: subq $16,%r11 - vmovdqu (%rsi),%xmm0 + vmovdqu (%rsi),%xmm0 // next data block vpclmulqdq $0x00,(%rdi,%r11,1),%xmm0,%xmm6 vpxor %xmm6,%xmm3,%xmm3 @@ -189,10 +189,10 @@ jmp .Lhtable_polyval_main_loop .Lhtable_polyval_no_prefix: - - - - +// At this point we know the number of blocks is a multiple of 8. However, +// the reduction in the main loop includes a multiplication by x^(-128). In +// order to counter this, the existing tag needs to be multiplied by x^128. +// In practice, this just means that it is loaded into %xmm9, not %xmm1. vpxor %xmm1,%xmm1,%xmm1 vmovdqa (%rcx),%xmm9 @@ -201,7 +201,7 @@ subq $0x80,%rdx jb .Lhtable_polyval_out - vmovdqu 112(%rsi),%xmm0 + vmovdqu 112(%rsi),%xmm0 // Ii vpclmulqdq $0x01,(%rdi),%xmm0,%xmm5 vpclmulqdq $0x00,(%rdi),%xmm0,%xmm3 @@ -209,7 +209,7 @@ vpclmulqdq $0x10,(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 96(%rsi),%xmm0 vpclmulqdq $0x01,16(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 @@ -221,10 +221,10 @@ vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 80(%rsi),%xmm0 - vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 + vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 // reduction stage 1a vpalignr $8,%xmm1,%xmm1,%xmm1 vpclmulqdq $0x01,32(%rdi),%xmm0,%xmm6 @@ -237,8 +237,8 @@ vpxor %xmm6,%xmm5,%xmm5 - vpxor %xmm7,%xmm1,%xmm1 - + vpxor %xmm7,%xmm1,%xmm1 // reduction stage 1b +// ######################################################## vmovdqu 64(%rsi),%xmm0 vpclmulqdq $0x01,48(%rdi),%xmm0,%xmm6 @@ -250,10 +250,10 @@ vpclmulqdq $0x10,48(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 48(%rsi),%xmm0 - vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 + vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm7 // reduction stage 2a vpalignr $8,%xmm1,%xmm1,%xmm1 vpclmulqdq $0x01,64(%rdi),%xmm0,%xmm6 @@ -266,8 +266,8 @@ vpxor %xmm6,%xmm5,%xmm5 - vpxor %xmm7,%xmm1,%xmm1 - + vpxor %xmm7,%xmm1,%xmm1 // reduction stage 2b +// ######################################################## vmovdqu 32(%rsi),%xmm0 vpclmulqdq $0x01,80(%rdi),%xmm0,%xmm6 @@ -280,8 +280,8 @@ vpxor %xmm6,%xmm5,%xmm5 - vpxor %xmm9,%xmm1,%xmm1 - + vpxor %xmm9,%xmm1,%xmm1 // reduction finalize +// ######################################################## vmovdqu 16(%rsi),%xmm0 vpclmulqdq $0x01,96(%rdi),%xmm0,%xmm6 @@ -293,7 +293,7 @@ vpclmulqdq $0x10,96(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vmovdqu 0(%rsi),%xmm0 vpxor %xmm1,%xmm0,%xmm0 @@ -306,7 +306,7 @@ vpclmulqdq $0x10,112(%rdi),%xmm0,%xmm6 vpxor %xmm6,%xmm5,%xmm5 - +// ######################################################## vpsrldq $8,%xmm5,%xmm6 vpslldq $8,%xmm5,%xmm5 @@ -316,7 +316,7 @@ leaq 128(%rsi),%rsi jmp .Lhtable_polyval_main_loop - +// ######################################################## .Lhtable_polyval_out: vpclmulqdq $0x10,poly(%rip),%xmm1,%xmm6 @@ -345,24 +345,24 @@ ret .Lpolyval_horner_start: - - +// We will start with L GFMULS for POLYVAL(BIG_BUFFER) +// RES = GFMUL(RES, H) xorq %r10,%r10 - shlq $4,%rcx + shlq $4,%rcx // L contains number of bytes to process vmovdqa (%rsi),%xmm1 vmovdqa (%rdi),%xmm0 .Lpolyval_horner_loop: - vpxor (%rdx,%r10,1),%xmm0,%xmm0 - call GFMUL + vpxor (%rdx,%r10,1),%xmm0,%xmm0 // RES = RES + Xi + call GFMUL // RES = RES * H addq $16,%r10 cmpq %r10,%rcx jne .Lpolyval_horner_loop - +// calculation of T is complete. RES=T vmovdqa %xmm0,(%rdi) ret .cfi_endproc @@ -374,8 +374,8 @@ aes128gcmsiv_aes_ks: .cfi_startproc _CET_ENDBR - vmovdqu (%rdi),%xmm1 - vmovdqa %xmm1,(%rsi) + vmovdqu (%rdi),%xmm1 // xmm1 = user key + vmovdqa %xmm1,(%rsi) // rsi points to output vmovdqa con1(%rip),%xmm0 vmovdqa mask(%rip),%xmm15 @@ -383,9 +383,9 @@ movq $8,%rax .Lks128_loop: - addq $16,%rsi + addq $16,%rsi // rsi points for next key subq $1,%rax - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // xmm2 = shuffled user key vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 vpslldq $4,%xmm1,%xmm3 @@ -479,19 +479,19 @@ aes128gcmsiv_aes_ks_enc_x1: .cfi_startproc _CET_ENDBR - vmovdqa (%rcx),%xmm1 + vmovdqa (%rcx),%xmm1 // xmm1 = first 16 bytes of random key vmovdqa 0(%rdi),%xmm4 - vmovdqa %xmm1,(%rdx) + vmovdqa %xmm1,(%rdx) // KEY[0] = first 16 bytes of random key vpxor %xmm1,%xmm4,%xmm4 - vmovdqa con1(%rip),%xmm0 - vmovdqa mask(%rip),%xmm15 + vmovdqa con1(%rip),%xmm0 // xmm0 = 1,1,1,1 + vmovdqa mask(%rip),%xmm15 // xmm15 = mask - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -500,10 +500,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,16(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -512,10 +512,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,32(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -524,10 +524,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,48(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -536,10 +536,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,64(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -548,10 +548,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,80(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -560,10 +560,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,96(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -572,10 +572,10 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,112(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -587,10 +587,10 @@ vmovdqa con2(%rip),%xmm0 - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 vpslld $1,%xmm0,%xmm0 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -599,9 +599,9 @@ vaesenc %xmm1,%xmm4,%xmm4 vmovdqa %xmm1,144(%rdx) - vpshufb %xmm15,%xmm1,%xmm2 + vpshufb %xmm15,%xmm1,%xmm2 // vaesenclast %xmm0,%xmm2,%xmm2 - vpsllq $32,%xmm1,%xmm3 + vpsllq $32,%xmm1,%xmm3 // vpxor %xmm3,%xmm1,%xmm1 vpshufb con3(%rip),%xmm1,%xmm3 vpxor %xmm3,%xmm1,%xmm1 @@ -622,11 +622,11 @@ aes128gcmsiv_kdf: .cfi_startproc _CET_ENDBR +// parameter 1: %rdi Pointer to NONCE +// parameter 2: %rsi Pointer to CT +// parameter 4: %rdx Pointer to keys - - - - vmovdqa (%rdx),%xmm1 + vmovdqa (%rdx),%xmm1 // xmm1 = first 16 bytes of random key vmovdqa 0(%rdi),%xmm9 vmovdqa and_mask(%rip),%xmm12 vmovdqa one(%rip),%xmm13 @@ -728,20 +728,20 @@ .cfi_adjust_cfa_offset 8 .cfi_offset %r13,-24 - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks movq %r8,%r10 shlq $62,%r10 shrq $62,%r10 - +// make IV from TAG vmovdqa (%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // IV = [1]TAG[126...32][00..00] - vmovdqu four(%rip),%xmm4 - vmovdqa %xmm15,%xmm0 - vpaddd one(%rip),%xmm15,%xmm1 - vpaddd two(%rip),%xmm15,%xmm2 - vpaddd three(%rip),%xmm15,%xmm3 + vmovdqu four(%rip),%xmm4 // Register to increment counters + vmovdqa %xmm15,%xmm0 // CTR1 = TAG[1][127...32][00..00] + vpaddd one(%rip),%xmm15,%xmm1 // CTR2 = TAG[1][127...32][00..01] + vpaddd two(%rip),%xmm15,%xmm2 // CTR3 = TAG[1][127...32][00..02] + vpaddd three(%rip),%xmm15,%xmm3 // CTR4 = TAG[1][127...32][00..03] shrq $2,%r8 je .L128_enc_msg_x4_check_remainder @@ -829,7 +829,7 @@ vaesenclast %xmm12,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm5,%xmm5 vpxor 16(%rdi),%xmm6,%xmm6 vpxor 32(%rdi),%xmm7,%xmm7 @@ -852,10 +852,10 @@ je .L128_enc_msg_x4_out .L128_enc_msg_x4_loop2: - - +// enc each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm5 - vpaddd one(%rip),%xmm0,%xmm0 + vpaddd one(%rip),%xmm0,%xmm0 // inc counter vpxor (%rcx),%xmm5,%xmm5 vaesenc 16(%rcx),%xmm5,%xmm5 @@ -869,7 +869,7 @@ vaesenc 144(%rcx),%xmm5,%xmm5 vaesenclast 160(%rcx),%xmm5,%xmm5 - +// XOR with plaintext vpxor (%rdi),%xmm5,%xmm5 vmovdqu %xmm5,(%rsi) @@ -913,29 +913,29 @@ movq %rsp,%rbp .cfi_def_cfa_register rbp - +// Place in stack subq $128,%rsp andq $-64,%rsp - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks movq %r8,%r10 shlq $61,%r10 shrq $61,%r10 - +// make IV from TAG vmovdqu (%rdx),%xmm1 - vpor OR_MASK(%rip),%xmm1,%xmm1 + vpor OR_MASK(%rip),%xmm1,%xmm1 // TMP1= IV = [1]TAG[126...32][00..00] - +// store counter8 in the stack vpaddd seven(%rip),%xmm1,%xmm0 - vmovdqu %xmm0,(%rsp) - vpaddd one(%rip),%xmm1,%xmm9 - vpaddd two(%rip),%xmm1,%xmm10 - vpaddd three(%rip),%xmm1,%xmm11 - vpaddd four(%rip),%xmm1,%xmm12 - vpaddd five(%rip),%xmm1,%xmm13 - vpaddd six(%rip),%xmm1,%xmm14 - vmovdqa %xmm1,%xmm0 + vmovdqu %xmm0,(%rsp) // CTR8 = TAG[127...32][00..07] + vpaddd one(%rip),%xmm1,%xmm9 // CTR2 = TAG[127...32][00..01] + vpaddd two(%rip),%xmm1,%xmm10 // CTR3 = TAG[127...32][00..02] + vpaddd three(%rip),%xmm1,%xmm11 // CTR4 = TAG[127...32][00..03] + vpaddd four(%rip),%xmm1,%xmm12 // CTR5 = TAG[127...32][00..04] + vpaddd five(%rip),%xmm1,%xmm13 // CTR6 = TAG[127...32][00..05] + vpaddd six(%rip),%xmm1,%xmm14 // CTR7 = TAG[127...32][00..06] + vmovdqa %xmm1,%xmm0 // CTR1 = TAG[127...32][00..00] shrq $3,%r8 je .L128_enc_msg_x8_check_remainder @@ -954,7 +954,7 @@ vmovdqa %xmm12,%xmm5 vmovdqa %xmm13,%xmm6 vmovdqa %xmm14,%xmm7 - +// move from stack vmovdqu (%rsp),%xmm8 vpxor (%rcx),%xmm1,%xmm1 @@ -976,7 +976,7 @@ vaesenc %xmm15,%xmm7,%xmm7 vaesenc %xmm15,%xmm8,%xmm8 - vmovdqu (%rsp),%xmm14 + vmovdqu (%rsp),%xmm14 // deal with CTR8 vpaddd eight(%rip),%xmm14,%xmm14 vmovdqu %xmm14,(%rsp) vmovdqu 32(%rcx),%xmm15 @@ -1077,7 +1077,7 @@ vaesenclast %xmm15,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm1,%xmm1 vpxor 16(%rdi),%xmm2,%xmm2 vpxor 32(%rdi),%xmm3,%xmm3 @@ -1108,10 +1108,10 @@ je .L128_enc_msg_x8_out .L128_enc_msg_x8_loop2: - - +// enc each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm1 - vpaddd one(%rip),%xmm0,%xmm0 + vpaddd one(%rip),%xmm0,%xmm0 // inc counter vpxor (%rcx),%xmm1,%xmm1 vaesenc 16(%rcx),%xmm1,%xmm1 @@ -1125,7 +1125,7 @@ vaesenc 144(%rcx),%xmm1,%xmm1 vaesenclast 160(%rcx),%xmm1,%xmm1 - +// XOR with Plaintext vpxor (%rdi),%xmm1,%xmm1 vmovdqu %xmm1,(%rsi) @@ -1165,10 +1165,10 @@ .L128_dec_start: vzeroupper vmovdqa (%rdx),%xmm0 - - +// The claimed tag is provided after the current calculated tag value. +// CTRBLKs is made from it. vmovdqu 16(%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // CTR = [1]TAG[126...32][00..00] movq %rdx,%rax leaq 32(%rax),%rax @@ -1176,11 +1176,11 @@ andq $~15,%r9 - +// If less then 6 blocks, make singles cmpq $96,%r9 jb .L128_dec_loop2 - +// Decrypt the first six blocks subq $96,%r9 vmovdqa %xmm15,%xmm7 vpaddd one(%rip),%xmm7,%xmm8 @@ -1277,7 +1277,7 @@ vaesenclast %xmm4,%xmm11,%xmm11 vaesenclast %xmm4,%xmm12,%xmm12 - +// XOR with CT vpxor 0(%rdi),%xmm7,%xmm7 vpxor 16(%rdi),%xmm8,%xmm8 vpxor 32(%rdi),%xmm9,%xmm9 @@ -1296,7 +1296,7 @@ addq $96,%rsi jmp .L128_dec_loop1 - +// Decrypt 6 blocks each time while hashing previous 6 blocks .align 64 .L128_dec_loop1: cmpq $96,%r9 @@ -1608,9 +1608,9 @@ vpxor %xmm5,%xmm0,%xmm0 .L128_dec_loop2: +// Here we encrypt any remaining whole block - - +// if there are no whole blocks cmpq $16,%r9 jb .L128_dec_out subq $16,%r9 @@ -1664,7 +1664,7 @@ vaesenc 112(%rdx),%xmm1,%xmm1 vaesenc 128(%rdx),%xmm1,%xmm1 vaesenc 144(%rdx),%xmm1,%xmm1 - vaesenclast 160(%rdx),%xmm1,%xmm1 + vaesenclast 160(%rdx),%xmm1,%xmm1 // STATE_1 == IV vmovdqa %xmm1,(%rsi) @@ -1678,14 +1678,14 @@ aes256gcmsiv_aes_ks_enc_x1: .cfi_startproc _CET_ENDBR - vmovdqa con1(%rip),%xmm0 - vmovdqa mask(%rip),%xmm15 + vmovdqa con1(%rip),%xmm0 // CON_MASK = 1,1,1,1 + vmovdqa mask(%rip),%xmm15 // MASK_256 vmovdqa (%rdi),%xmm8 - vmovdqa (%rcx),%xmm1 + vmovdqa (%rcx),%xmm1 // KEY_1 || KEY_2 [0..7] = user key vmovdqa 16(%rcx),%xmm3 vpxor %xmm1,%xmm8,%xmm8 vaesenc %xmm3,%xmm8,%xmm8 - vmovdqu %xmm1,(%rdx) + vmovdqu %xmm1,(%rdx) // First round key vmovdqu %xmm3,16(%rdx) vpxor %xmm14,%xmm14,%xmm14 @@ -1877,7 +1877,7 @@ vaesenc 176(%rdx),%xmm1,%xmm1 vaesenc 192(%rdx),%xmm1,%xmm1 vaesenc 208(%rdx),%xmm1,%xmm1 - vaesenclast 224(%rdx),%xmm1,%xmm1 + vaesenclast 224(%rdx),%xmm1,%xmm1 // %xmm1 == IV vmovdqa %xmm1,(%rsi) ret .cfi_endproc @@ -1895,7 +1895,7 @@ .L256_enc_msg_x4_start: movq %r8,%r10 - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks shlq $60,%r10 jz .L256_enc_msg_x4_start2 addq $1,%r8 @@ -1905,15 +1905,15 @@ shlq $62,%r10 shrq $62,%r10 - +// make IV from TAG vmovdqa (%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // IV = [1]TAG[126...32][00..00] - vmovdqa four(%rip),%xmm4 - vmovdqa %xmm15,%xmm0 - vpaddd one(%rip),%xmm15,%xmm1 - vpaddd two(%rip),%xmm15,%xmm2 - vpaddd three(%rip),%xmm15,%xmm3 + vmovdqa four(%rip),%xmm4 // Register to increment counters + vmovdqa %xmm15,%xmm0 // CTR1 = TAG[1][127...32][00..00] + vpaddd one(%rip),%xmm15,%xmm1 // CTR2 = TAG[1][127...32][00..01] + vpaddd two(%rip),%xmm15,%xmm2 // CTR3 = TAG[1][127...32][00..02] + vpaddd three(%rip),%xmm15,%xmm3 // CTR4 = TAG[1][127...32][00..03] shrq $2,%r8 je .L256_enc_msg_x4_check_remainder @@ -2025,7 +2025,7 @@ vaesenclast %xmm12,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm5,%xmm5 vpxor 16(%rdi),%xmm6,%xmm6 vpxor 32(%rdi),%xmm7,%xmm7 @@ -2048,11 +2048,11 @@ je .L256_enc_msg_x4_out .L256_enc_msg_x4_loop2: - - +// encrypt each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm5 - vpaddd one(%rip),%xmm0,%xmm0 + vpaddd one(%rip),%xmm0,%xmm0 // inc counter vpxor (%rcx),%xmm5,%xmm5 vaesenc 16(%rcx),%xmm5,%xmm5 vaesenc 32(%rcx),%xmm5,%xmm5 @@ -2069,7 +2069,7 @@ vaesenc 208(%rcx),%xmm5,%xmm5 vaesenclast 224(%rcx),%xmm5,%xmm5 - +// XOR with Plaintext vpxor (%rdi),%xmm5,%xmm5 vmovdqu %xmm5,(%rsi) @@ -2096,13 +2096,13 @@ ret .L256_enc_msg_x8_start: - +// Place in stack movq %rsp,%r11 subq $16,%r11 andq $-64,%r11 movq %r8,%r10 - shrq $4,%r8 + shrq $4,%r8 // LEN = num of blocks shlq $60,%r10 jz .L256_enc_msg_x8_start2 addq $1,%r8 @@ -2112,20 +2112,20 @@ shlq $61,%r10 shrq $61,%r10 - +// Make IV from TAG vmovdqa (%rdx),%xmm1 - vpor OR_MASK(%rip),%xmm1,%xmm1 + vpor OR_MASK(%rip),%xmm1,%xmm1 // TMP1= IV = [1]TAG[126...32][00..00] - +// store counter8 on the stack vpaddd seven(%rip),%xmm1,%xmm0 - vmovdqa %xmm0,(%r11) - vpaddd one(%rip),%xmm1,%xmm9 - vpaddd two(%rip),%xmm1,%xmm10 - vpaddd three(%rip),%xmm1,%xmm11 - vpaddd four(%rip),%xmm1,%xmm12 - vpaddd five(%rip),%xmm1,%xmm13 - vpaddd six(%rip),%xmm1,%xmm14 - vmovdqa %xmm1,%xmm0 + vmovdqa %xmm0,(%r11) // CTR8 = TAG[127...32][00..07] + vpaddd one(%rip),%xmm1,%xmm9 // CTR2 = TAG[127...32][00..01] + vpaddd two(%rip),%xmm1,%xmm10 // CTR3 = TAG[127...32][00..02] + vpaddd three(%rip),%xmm1,%xmm11 // CTR4 = TAG[127...32][00..03] + vpaddd four(%rip),%xmm1,%xmm12 // CTR5 = TAG[127...32][00..04] + vpaddd five(%rip),%xmm1,%xmm13 // CTR6 = TAG[127...32][00..05] + vpaddd six(%rip),%xmm1,%xmm14 // CTR7 = TAG[127...32][00..06] + vmovdqa %xmm1,%xmm0 // CTR1 = TAG[127...32][00..00] shrq $3,%r8 jz .L256_enc_msg_x8_check_remainder @@ -2144,7 +2144,7 @@ vmovdqa %xmm12,%xmm5 vmovdqa %xmm13,%xmm6 vmovdqa %xmm14,%xmm7 - +// move from stack vmovdqa (%r11),%xmm8 vpxor (%rcx),%xmm1,%xmm1 @@ -2166,7 +2166,7 @@ vaesenc %xmm15,%xmm7,%xmm7 vaesenc %xmm15,%xmm8,%xmm8 - vmovdqa (%r11),%xmm14 + vmovdqa (%r11),%xmm14 // deal with CTR8 vpaddd eight(%rip),%xmm14,%xmm14 vmovdqa %xmm14,(%r11) vmovdqu 32(%rcx),%xmm15 @@ -2307,7 +2307,7 @@ vaesenclast %xmm15,%xmm8,%xmm8 - +// XOR with Plaintext vpxor 0(%rdi),%xmm1,%xmm1 vpxor 16(%rdi),%xmm2,%xmm2 vpxor 32(%rdi),%xmm3,%xmm3 @@ -2338,8 +2338,8 @@ je .L256_enc_msg_x8_out .L256_enc_msg_x8_loop2: - - +// encrypt each block separately +// CTR1 is the highest counter (even if no LOOP done) vmovdqa %xmm0,%xmm1 vpaddd one(%rip),%xmm0,%xmm0 @@ -2359,7 +2359,7 @@ vaesenc 208(%rcx),%xmm1,%xmm1 vaesenclast 224(%rcx),%xmm1,%xmm1 - +// XOR with Plaintext vpxor (%rdi),%xmm1,%xmm1 vmovdqu %xmm1,(%rsi) @@ -2388,10 +2388,10 @@ .L256_dec_start: vzeroupper vmovdqa (%rdx),%xmm0 - - +// The claimed tag is provided after the current calculated tag value. +// CTRBLKs is made from it. vmovdqu 16(%rdx),%xmm15 - vpor OR_MASK(%rip),%xmm15,%xmm15 + vpor OR_MASK(%rip),%xmm15,%xmm15 // CTR = [1]TAG[126...32][00..00] movq %rdx,%rax leaq 32(%rax),%rax @@ -2399,11 +2399,11 @@ andq $~15,%r9 - +// If less then 6 blocks, make singles cmpq $96,%r9 jb .L256_dec_loop2 - +// Decrypt the first six blocks subq $96,%r9 vmovdqa %xmm15,%xmm7 vpaddd one(%rip),%xmm7,%xmm8 @@ -2532,7 +2532,7 @@ vaesenclast %xmm4,%xmm11,%xmm11 vaesenclast %xmm4,%xmm12,%xmm12 - +// XOR with CT vpxor 0(%rdi),%xmm7,%xmm7 vpxor 16(%rdi),%xmm8,%xmm8 vpxor 32(%rdi),%xmm9,%xmm9 @@ -2551,7 +2551,7 @@ addq $96,%rsi jmp .L256_dec_loop1 - +// Decrypt 6 blocks each time while hashing previous 6 blocks .align 64 .L256_dec_loop1: cmpq $96,%r9 @@ -2895,9 +2895,9 @@ vpxor %xmm5,%xmm0,%xmm0 .L256_dec_loop2: +// Here we encrypt any remaining whole block - - +// if there are no whole blocks cmpq $16,%r9 jb .L256_dec_out subq $16,%r9 @@ -2943,11 +2943,11 @@ aes256gcmsiv_kdf: .cfi_startproc _CET_ENDBR +// parameter 1: %rdi Pointer to NONCE +// parameter 2: %rsi Pointer to CT +// parameter 4: %rdx Pointer to keys - - - - vmovdqa (%rdx),%xmm1 + vmovdqa (%rdx),%xmm1 // xmm1 = first 16 bytes of random key vmovdqa 0(%rdi),%xmm4 vmovdqa and_mask(%rip),%xmm11 vmovdqa one(%rip),%xmm8
diff --git a/gen/crypto/aes128gcmsiv-x86_64-win.asm b/gen/crypto/aes128gcmsiv-x86_64-win.asm index e875c0a..2241677 100644 --- a/gen/crypto/aes128gcmsiv-x86_64-win.asm +++ b/gen/crypto/aes128gcmsiv-x86_64-win.asm
@@ -89,21 +89,21 @@ _CET_ENDBR vmovdqa xmm0,XMMWORD[rsi] vmovdqa xmm1,xmm0 - vmovdqa XMMWORD[rdi],xmm0 + vmovdqa XMMWORD[rdi],xmm0 ; H call GFMUL - vmovdqa XMMWORD[16+rdi],xmm0 + vmovdqa XMMWORD[16+rdi],xmm0 ; H^2 call GFMUL - vmovdqa XMMWORD[32+rdi],xmm0 + vmovdqa XMMWORD[32+rdi],xmm0 ; H^3 call GFMUL - vmovdqa XMMWORD[48+rdi],xmm0 + vmovdqa XMMWORD[48+rdi],xmm0 ; H^4 call GFMUL - vmovdqa XMMWORD[64+rdi],xmm0 + vmovdqa XMMWORD[64+rdi],xmm0 ; H^5 call GFMUL - vmovdqa XMMWORD[80+rdi],xmm0 + vmovdqa XMMWORD[80+rdi],xmm0 ; H^6 call GFMUL - vmovdqa XMMWORD[96+rdi],xmm0 + vmovdqa XMMWORD[96+rdi],xmm0 ; H^7 call GFMUL - vmovdqa XMMWORD[112+rdi],xmm0 + vmovdqa XMMWORD[112+rdi],xmm0 ; H^8 mov rdi,QWORD[8+rsp] ;WIN64 epilogue mov rsi,QWORD[16+rsp] ret @@ -125,17 +125,17 @@ _CET_ENDBR vmovdqa xmm0,XMMWORD[rsi] vmovdqa xmm1,xmm0 - vmovdqa XMMWORD[rdi],xmm0 + vmovdqa XMMWORD[rdi],xmm0 ; H call GFMUL - vmovdqa XMMWORD[16+rdi],xmm0 + vmovdqa XMMWORD[16+rdi],xmm0 ; H^2 call GFMUL - vmovdqa XMMWORD[32+rdi],xmm0 + vmovdqa XMMWORD[32+rdi],xmm0 ; H^3 call GFMUL - vmovdqa XMMWORD[48+rdi],xmm0 + vmovdqa XMMWORD[48+rdi],xmm0 ; H^4 call GFMUL - vmovdqa XMMWORD[64+rdi],xmm0 + vmovdqa XMMWORD[64+rdi],xmm0 ; H^5 call GFMUL - vmovdqa XMMWORD[80+rdi],xmm0 + vmovdqa XMMWORD[80+rdi],xmm0 ; H^6 mov rdi,QWORD[8+rsp] ;WIN64 epilogue mov rsi,QWORD[16+rsp] ret @@ -166,8 +166,8 @@ $L$htable_polyval_start: vzeroall - - +; We hash 8 blocks each iteration. If the total number of blocks is not a +; multiple of 8, we first hash the leading n%8 blocks. mov r11,rdx and r11,127 @@ -179,7 +179,7 @@ sub r11,16 - +; hash first prefix block vmovdqu xmm0,XMMWORD[rsi] vpxor xmm0,xmm0,xmm1 @@ -194,12 +194,12 @@ jnz NEAR $L$htable_polyval_prefix_loop jmp NEAR $L$htable_polyval_prefix_complete - +; hash remaining prefix blocks (up to 7 total prefix blocks) ALIGN 64 $L$htable_polyval_prefix_loop: sub r11,16 - vmovdqu xmm0,XMMWORD[rsi] + vmovdqu xmm0,XMMWORD[rsi] ; next data block vpclmulqdq xmm6,xmm0,XMMWORD[r11*1+rdi],0x00 vpxor xmm3,xmm3,xmm6 @@ -226,10 +226,10 @@ jmp NEAR $L$htable_polyval_main_loop $L$htable_polyval_no_prefix: - - - - +; At this point we know the number of blocks is a multiple of 8. However, +; the reduction in the main loop includes a multiplication by x^(-128). In +; order to counter this, the existing tag needs to be multiplied by x^128. +; In practice, this just means that it is loaded into %xmm9, not %xmm1. vpxor xmm1,xmm1,xmm1 vmovdqa xmm9,XMMWORD[rcx] @@ -238,7 +238,7 @@ sub rdx,0x80 jb NEAR $L$htable_polyval_out - vmovdqu xmm0,XMMWORD[112+rsi] + vmovdqu xmm0,XMMWORD[112+rsi] ; Ii vpclmulqdq xmm5,xmm0,XMMWORD[rdi],0x01 vpclmulqdq xmm3,xmm0,XMMWORD[rdi],0x00 @@ -246,7 +246,7 @@ vpclmulqdq xmm6,xmm0,XMMWORD[rdi],0x10 vpxor xmm5,xmm5,xmm6 - +; ######################################################## vmovdqu xmm0,XMMWORD[96+rsi] vpclmulqdq xmm6,xmm0,XMMWORD[16+rdi],0x01 vpxor xmm5,xmm5,xmm6 @@ -258,10 +258,10 @@ vpxor xmm5,xmm5,xmm6 - +; ######################################################## vmovdqu xmm0,XMMWORD[80+rsi] - vpclmulqdq xmm7,xmm1,XMMWORD[poly],0x10 + vpclmulqdq xmm7,xmm1,XMMWORD[poly],0x10 ; reduction stage 1a vpalignr xmm1,xmm1,xmm1,8 vpclmulqdq xmm6,xmm0,XMMWORD[32+rdi],0x01 @@ -274,8 +274,8 @@ vpxor xmm5,xmm5,xmm6 - vpxor xmm1,xmm1,xmm7 - + vpxor xmm1,xmm1,xmm7 ; reduction stage 1b +; ######################################################## vmovdqu xmm0,XMMWORD[64+rsi] vpclmulqdq xmm6,xmm0,XMMWORD[48+rdi],0x01 @@ -287,10 +287,10 @@ vpclmulqdq xmm6,xmm0,XMMWORD[48+rdi],0x10 vpxor xmm5,xmm5,xmm6 - +; ######################################################## vmovdqu xmm0,XMMWORD[48+rsi] - vpclmulqdq xmm7,xmm1,XMMWORD[poly],0x10 + vpclmulqdq xmm7,xmm1,XMMWORD[poly],0x10 ; reduction stage 2a vpalignr xmm1,xmm1,xmm1,8 vpclmulqdq xmm6,xmm0,XMMWORD[64+rdi],0x01 @@ -303,8 +303,8 @@ vpxor xmm5,xmm5,xmm6 - vpxor xmm1,xmm1,xmm7 - + vpxor xmm1,xmm1,xmm7 ; reduction stage 2b +; ######################################################## vmovdqu xmm0,XMMWORD[32+rsi] vpclmulqdq xmm6,xmm0,XMMWORD[80+rdi],0x01 @@ -317,8 +317,8 @@ vpxor xmm5,xmm5,xmm6 - vpxor xmm1,xmm1,xmm9 - + vpxor xmm1,xmm1,xmm9 ; reduction finalize +; ######################################################## vmovdqu xmm0,XMMWORD[16+rsi] vpclmulqdq xmm6,xmm0,XMMWORD[96+rdi],0x01 @@ -330,7 +330,7 @@ vpclmulqdq xmm6,xmm0,XMMWORD[96+rdi],0x10 vpxor xmm5,xmm5,xmm6 - +; ######################################################## vmovdqu xmm0,XMMWORD[rsi] vpxor xmm0,xmm0,xmm1 @@ -343,7 +343,7 @@ vpclmulqdq xmm6,xmm0,XMMWORD[112+rdi],0x10 vpxor xmm5,xmm5,xmm6 - +; ######################################################## vpsrldq xmm6,xmm5,8 vpslldq xmm5,xmm5,8 @@ -353,7 +353,7 @@ lea rsi,[128+rsi] jmp NEAR $L$htable_polyval_main_loop - +; ######################################################## $L$htable_polyval_out: vpclmulqdq xmm6,xmm1,XMMWORD[poly],0x10 @@ -395,24 +395,24 @@ ret $L$polyval_horner_start: - - +; We will start with L GFMULS for POLYVAL(BIG_BUFFER) +; RES = GFMUL(RES, H) xor r10,r10 - shl rcx,4 + shl rcx,4 ; L contains number of bytes to process vmovdqa xmm1,XMMWORD[rsi] vmovdqa xmm0,XMMWORD[rdi] $L$polyval_horner_loop: - vpxor xmm0,xmm0,XMMWORD[r10*1+rdx] - call GFMUL + vpxor xmm0,xmm0,XMMWORD[r10*1+rdx] ; RES = RES + Xi + call GFMUL ; RES = RES * H add r10,16 cmp rcx,r10 jne NEAR $L$polyval_horner_loop - +; calculation of T is complete. RES=T vmovdqa XMMWORD[rdi],xmm0 mov rdi,QWORD[8+rsp] ;WIN64 epilogue mov rsi,QWORD[16+rsp] @@ -433,8 +433,8 @@ _CET_ENDBR - vmovdqu xmm1,XMMWORD[rdi] - vmovdqa XMMWORD[rsi],xmm1 + vmovdqu xmm1,XMMWORD[rdi] ; xmm1 = user key + vmovdqa XMMWORD[rsi],xmm1 ; rsi points to output vmovdqa xmm0,XMMWORD[con1] vmovdqa xmm15,XMMWORD[mask] @@ -442,9 +442,9 @@ mov rax,8 $L$ks128_loop: - add rsi,16 + add rsi,16 ; rsi points for next key sub rax,1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; xmm2 = shuffled user key vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 vpslldq xmm3,xmm1,4 @@ -558,19 +558,19 @@ _CET_ENDBR - vmovdqa xmm1,XMMWORD[rcx] + vmovdqa xmm1,XMMWORD[rcx] ; xmm1 = first 16 bytes of random key vmovdqa xmm4,XMMWORD[rdi] - vmovdqa XMMWORD[rdx],xmm1 + vmovdqa XMMWORD[rdx],xmm1 ; KEY[0] = first 16 bytes of random key vpxor xmm4,xmm4,xmm1 - vmovdqa xmm0,XMMWORD[con1] - vmovdqa xmm15,XMMWORD[mask] + vmovdqa xmm0,XMMWORD[con1] ; xmm0 = 1,1,1,1 + vmovdqa xmm15,XMMWORD[mask] ; xmm15 = mask - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -579,10 +579,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[16+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -591,10 +591,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[32+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -603,10 +603,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[48+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -615,10 +615,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[64+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -627,10 +627,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[80+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -639,10 +639,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[96+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -651,10 +651,10 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[112+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -666,10 +666,10 @@ vmovdqa xmm0,XMMWORD[con2] - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 vpslld xmm0,xmm0,1 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -678,9 +678,9 @@ vaesenc xmm4,xmm4,xmm1 vmovdqa XMMWORD[144+rdx],xmm1 - vpshufb xmm2,xmm1,xmm15 + vpshufb xmm2,xmm1,xmm15 ; !!saving mov instruction to xmm2 vaesenclast xmm2,xmm2,xmm0 - vpsllq xmm3,xmm1,32 + vpsllq xmm3,xmm1,32 ; !!saving mov instruction to xmm3 vpxor xmm1,xmm1,xmm3 vpshufb xmm3,xmm1,XMMWORD[con3] vpxor xmm1,xmm1,xmm3 @@ -711,11 +711,11 @@ _CET_ENDBR +; parameter 1: %rdi Pointer to NONCE +; parameter 2: %rsi Pointer to CT +; parameter 4: %rdx Pointer to keys - - - - vmovdqa xmm1,XMMWORD[rdx] + vmovdqa xmm1,XMMWORD[rdx] ; xmm1 = first 16 bytes of random key vmovdqa xmm9,XMMWORD[rdi] vmovdqa xmm12,XMMWORD[and_mask] vmovdqa xmm13,XMMWORD[one] @@ -829,20 +829,20 @@ push r13 - shr r8,4 + shr r8,4 ; LEN = num of blocks mov r10,r8 shl r10,62 shr r10,62 - +; make IV from TAG vmovdqa xmm15,XMMWORD[rdx] - vpor xmm15,xmm15,XMMWORD[OR_MASK] + vpor xmm15,xmm15,XMMWORD[OR_MASK] ; IV = [1]TAG[126...32][00..00] - vmovdqu xmm4,XMMWORD[four] - vmovdqa xmm0,xmm15 - vpaddd xmm1,xmm15,XMMWORD[one] - vpaddd xmm2,xmm15,XMMWORD[two] - vpaddd xmm3,xmm15,XMMWORD[three] + vmovdqu xmm4,XMMWORD[four] ; Register to increment counters + vmovdqa xmm0,xmm15 ; CTR1 = TAG[1][127...32][00..00] + vpaddd xmm1,xmm15,XMMWORD[one] ; CTR2 = TAG[1][127...32][00..01] + vpaddd xmm2,xmm15,XMMWORD[two] ; CTR3 = TAG[1][127...32][00..02] + vpaddd xmm3,xmm15,XMMWORD[three] ; CTR4 = TAG[1][127...32][00..03] shr r8,2 je NEAR $L$128_enc_msg_x4_check_remainder @@ -930,7 +930,7 @@ vaesenclast xmm8,xmm8,xmm12 - +; XOR with Plaintext vpxor xmm5,xmm5,XMMWORD[rdi] vpxor xmm6,xmm6,XMMWORD[16+rdi] vpxor xmm7,xmm7,XMMWORD[32+rdi] @@ -953,10 +953,10 @@ je NEAR $L$128_enc_msg_x4_out $L$128_enc_msg_x4_loop2: - - +; enc each block separately +; CTR1 is the highest counter (even if no LOOP done) vmovdqa xmm5,xmm0 - vpaddd xmm0,xmm0,XMMWORD[one] + vpaddd xmm0,xmm0,XMMWORD[one] ; inc counter vpxor xmm5,xmm5,XMMWORD[rcx] vaesenc xmm5,xmm5,XMMWORD[16+rcx] @@ -970,7 +970,7 @@ vaesenc xmm5,xmm5,XMMWORD[144+rcx] vaesenclast xmm5,xmm5,XMMWORD[160+rcx] - +; XOR with plaintext vpxor xmm5,xmm5,XMMWORD[rdi] vmovdqu XMMWORD[rsi],xmm5 @@ -1023,29 +1023,29 @@ mov rbp,rsp - +; Place in stack sub rsp,128 and rsp,-64 - shr r8,4 + shr r8,4 ; LEN = num of blocks mov r10,r8 shl r10,61 shr r10,61 - +; make IV from TAG vmovdqu xmm1,XMMWORD[rdx] - vpor xmm1,xmm1,XMMWORD[OR_MASK] + vpor xmm1,xmm1,XMMWORD[OR_MASK] ; TMP1= IV = [1]TAG[126...32][00..00] - +; store counter8 in the stack vpaddd xmm0,xmm1,XMMWORD[seven] - vmovdqu XMMWORD[rsp],xmm0 - vpaddd xmm9,xmm1,XMMWORD[one] - vpaddd xmm10,xmm1,XMMWORD[two] - vpaddd xmm11,xmm1,XMMWORD[three] - vpaddd xmm12,xmm1,XMMWORD[four] - vpaddd xmm13,xmm1,XMMWORD[five] - vpaddd xmm14,xmm1,XMMWORD[six] - vmovdqa xmm0,xmm1 + vmovdqu XMMWORD[rsp],xmm0 ; CTR8 = TAG[127...32][00..07] + vpaddd xmm9,xmm1,XMMWORD[one] ; CTR2 = TAG[127...32][00..01] + vpaddd xmm10,xmm1,XMMWORD[two] ; CTR3 = TAG[127...32][00..02] + vpaddd xmm11,xmm1,XMMWORD[three] ; CTR4 = TAG[127...32][00..03] + vpaddd xmm12,xmm1,XMMWORD[four] ; CTR5 = TAG[127...32][00..04] + vpaddd xmm13,xmm1,XMMWORD[five] ; CTR6 = TAG[127...32][00..05] + vpaddd xmm14,xmm1,XMMWORD[six] ; CTR7 = TAG[127...32][00..06] + vmovdqa xmm0,xmm1 ; CTR1 = TAG[127...32][00..00] shr r8,3 je NEAR $L$128_enc_msg_x8_check_remainder @@ -1064,7 +1064,7 @@ vmovdqa xmm5,xmm12 vmovdqa xmm6,xmm13 vmovdqa xmm7,xmm14 - +; move from stack vmovdqu xmm8,XMMWORD[rsp] vpxor xmm1,xmm1,XMMWORD[rcx] @@ -1086,7 +1086,7 @@ vaesenc xmm7,xmm7,xmm15 vaesenc xmm8,xmm8,xmm15 - vmovdqu xmm14,XMMWORD[rsp] + vmovdqu xmm14,XMMWORD[rsp] ; deal with CTR8 vpaddd xmm14,xmm14,XMMWORD[eight] vmovdqu XMMWORD[rsp],xmm14 vmovdqu xmm15,XMMWORD[32+rcx] @@ -1187,7 +1187,7 @@ vaesenclast xmm8,xmm8,xmm15 - +; XOR with Plaintext vpxor xmm1,xmm1,XMMWORD[rdi] vpxor xmm2,xmm2,XMMWORD[16+rdi] vpxor xmm3,xmm3,XMMWORD[32+rdi] @@ -1218,10 +1218,10 @@ je NEAR $L$128_enc_msg_x8_out $L$128_enc_msg_x8_loop2: - - +; enc each block separately +; CTR1 is the highest counter (even if no LOOP done) vmovdqa xmm1,xmm0 - vpaddd xmm0,xmm0,XMMWORD[one] + vpaddd xmm0,xmm0,XMMWORD[one] ; inc counter vpxor xmm1,xmm1,XMMWORD[rcx] vaesenc xmm1,xmm1,XMMWORD[16+rcx] @@ -1235,7 +1235,7 @@ vaesenc xmm1,xmm1,XMMWORD[144+rcx] vaesenclast xmm1,xmm1,XMMWORD[160+rcx] - +; XOR with Plaintext vpxor xmm1,xmm1,XMMWORD[rdi] vmovdqu XMMWORD[rsi],xmm1 @@ -1287,10 +1287,10 @@ $L$128_dec_start: vzeroupper vmovdqa xmm0,XMMWORD[rdx] - - +; The claimed tag is provided after the current calculated tag value. +; CTRBLKs is made from it. vmovdqu xmm15,XMMWORD[16+rdx] - vpor xmm15,xmm15,XMMWORD[OR_MASK] + vpor xmm15,xmm15,XMMWORD[OR_MASK] ; CTR = [1]TAG[126...32][00..00] mov rax,rdx lea rax,[32+rax] @@ -1298,11 +1298,11 @@ and r9,~15 - +; If less then 6 blocks, make singles cmp r9,96 jb NEAR $L$128_dec_loop2 - +; Decrypt the first six blocks sub r9,96 vmovdqa xmm7,xmm15 vpaddd xmm8,xmm7,XMMWORD[one] @@ -1399,7 +1399,7 @@ vaesenclast xmm11,xmm11,xmm4 vaesenclast xmm12,xmm12,xmm4 - +; XOR with CT vpxor xmm7,xmm7,XMMWORD[rdi] vpxor xmm8,xmm8,XMMWORD[16+rdi] vpxor xmm9,xmm9,XMMWORD[32+rdi] @@ -1418,7 +1418,7 @@ add rsi,96 jmp NEAR $L$128_dec_loop1 - +; Decrypt 6 blocks each time while hashing previous 6 blocks ALIGN 64 $L$128_dec_loop1: cmp r9,96 @@ -1730,9 +1730,9 @@ vpxor xmm0,xmm0,xmm5 $L$128_dec_loop2: +; Here we encrypt any remaining whole block - - +; if there are no whole blocks cmp r9,16 jb NEAR $L$128_dec_out sub r9,16 @@ -1796,7 +1796,7 @@ vaesenc xmm1,xmm1,XMMWORD[112+rdx] vaesenc xmm1,xmm1,XMMWORD[128+rdx] vaesenc xmm1,xmm1,XMMWORD[144+rdx] - vaesenclast xmm1,xmm1,XMMWORD[160+rdx] + vaesenclast xmm1,xmm1,XMMWORD[160+rdx] ; STATE_1 == IV vmovdqa XMMWORD[rsi],xmm1 @@ -1821,14 +1821,14 @@ _CET_ENDBR - vmovdqa xmm0,XMMWORD[con1] - vmovdqa xmm15,XMMWORD[mask] + vmovdqa xmm0,XMMWORD[con1] ; CON_MASK = 1,1,1,1 + vmovdqa xmm15,XMMWORD[mask] ; MASK_256 vmovdqa xmm8,XMMWORD[rdi] - vmovdqa xmm1,XMMWORD[rcx] + vmovdqa xmm1,XMMWORD[rcx] ; KEY_1 || KEY_2 [0..7] = user key vmovdqa xmm3,XMMWORD[16+rcx] vpxor xmm8,xmm8,xmm1 vaesenc xmm8,xmm8,xmm3 - vmovdqu XMMWORD[rdx],xmm1 + vmovdqu XMMWORD[rdx],xmm1 ; First round key vmovdqu XMMWORD[16+rdx],xmm3 vpxor xmm14,xmm14,xmm14 @@ -2030,7 +2030,7 @@ vaesenc xmm1,xmm1,XMMWORD[176+rdx] vaesenc xmm1,xmm1,XMMWORD[192+rdx] vaesenc xmm1,xmm1,XMMWORD[208+rdx] - vaesenclast xmm1,xmm1,XMMWORD[224+rdx] + vaesenclast xmm1,xmm1,XMMWORD[224+rdx] ; %xmm1 == IV vmovdqa XMMWORD[rsi],xmm1 mov rdi,QWORD[8+rsp] ;WIN64 epilogue mov rsi,QWORD[16+rsp] @@ -2062,7 +2062,7 @@ $L$256_enc_msg_x4_start: mov r10,r8 - shr r8,4 + shr r8,4 ; LEN = num of blocks shl r10,60 jz NEAR $L$256_enc_msg_x4_start2 add r8,1 @@ -2072,15 +2072,15 @@ shl r10,62 shr r10,62 - +; make IV from TAG vmovdqa xmm15,XMMWORD[rdx] - vpor xmm15,xmm15,XMMWORD[OR_MASK] + vpor xmm15,xmm15,XMMWORD[OR_MASK] ; IV = [1]TAG[126...32][00..00] - vmovdqa xmm4,XMMWORD[four] - vmovdqa xmm0,xmm15 - vpaddd xmm1,xmm15,XMMWORD[one] - vpaddd xmm2,xmm15,XMMWORD[two] - vpaddd xmm3,xmm15,XMMWORD[three] + vmovdqa xmm4,XMMWORD[four] ; Register to increment counters + vmovdqa xmm0,xmm15 ; CTR1 = TAG[1][127...32][00..00] + vpaddd xmm1,xmm15,XMMWORD[one] ; CTR2 = TAG[1][127...32][00..01] + vpaddd xmm2,xmm15,XMMWORD[two] ; CTR3 = TAG[1][127...32][00..02] + vpaddd xmm3,xmm15,XMMWORD[three] ; CTR4 = TAG[1][127...32][00..03] shr r8,2 je NEAR $L$256_enc_msg_x4_check_remainder @@ -2192,7 +2192,7 @@ vaesenclast xmm8,xmm8,xmm12 - +; XOR with Plaintext vpxor xmm5,xmm5,XMMWORD[rdi] vpxor xmm6,xmm6,XMMWORD[16+rdi] vpxor xmm7,xmm7,XMMWORD[32+rdi] @@ -2215,11 +2215,11 @@ je NEAR $L$256_enc_msg_x4_out $L$256_enc_msg_x4_loop2: - - +; encrypt each block separately +; CTR1 is the highest counter (even if no LOOP done) vmovdqa xmm5,xmm0 - vpaddd xmm0,xmm0,XMMWORD[one] + vpaddd xmm0,xmm0,XMMWORD[one] ; inc counter vpxor xmm5,xmm5,XMMWORD[rcx] vaesenc xmm5,xmm5,XMMWORD[16+rcx] vaesenc xmm5,xmm5,XMMWORD[32+rcx] @@ -2236,7 +2236,7 @@ vaesenc xmm5,xmm5,XMMWORD[208+rcx] vaesenclast xmm5,xmm5,XMMWORD[224+rcx] - +; XOR with Plaintext vpxor xmm5,xmm5,XMMWORD[rdi] vmovdqu XMMWORD[rsi],xmm5 @@ -2277,13 +2277,13 @@ ret $L$256_enc_msg_x8_start: - +; Place in stack mov r11,rsp sub r11,16 and r11,-64 mov r10,r8 - shr r8,4 + shr r8,4 ; LEN = num of blocks shl r10,60 jz NEAR $L$256_enc_msg_x8_start2 add r8,1 @@ -2293,20 +2293,20 @@ shl r10,61 shr r10,61 - +; Make IV from TAG vmovdqa xmm1,XMMWORD[rdx] - vpor xmm1,xmm1,XMMWORD[OR_MASK] + vpor xmm1,xmm1,XMMWORD[OR_MASK] ; TMP1= IV = [1]TAG[126...32][00..00] - +; store counter8 on the stack vpaddd xmm0,xmm1,XMMWORD[seven] - vmovdqa XMMWORD[r11],xmm0 - vpaddd xmm9,xmm1,XMMWORD[one] - vpaddd xmm10,xmm1,XMMWORD[two] - vpaddd xmm11,xmm1,XMMWORD[three] - vpaddd xmm12,xmm1,XMMWORD[four] - vpaddd xmm13,xmm1,XMMWORD[five] - vpaddd xmm14,xmm1,XMMWORD[six] - vmovdqa xmm0,xmm1 + vmovdqa XMMWORD[r11],xmm0 ; CTR8 = TAG[127...32][00..07] + vpaddd xmm9,xmm1,XMMWORD[one] ; CTR2 = TAG[127...32][00..01] + vpaddd xmm10,xmm1,XMMWORD[two] ; CTR3 = TAG[127...32][00..02] + vpaddd xmm11,xmm1,XMMWORD[three] ; CTR4 = TAG[127...32][00..03] + vpaddd xmm12,xmm1,XMMWORD[four] ; CTR5 = TAG[127...32][00..04] + vpaddd xmm13,xmm1,XMMWORD[five] ; CTR6 = TAG[127...32][00..05] + vpaddd xmm14,xmm1,XMMWORD[six] ; CTR7 = TAG[127...32][00..06] + vmovdqa xmm0,xmm1 ; CTR1 = TAG[127...32][00..00] shr r8,3 jz NEAR $L$256_enc_msg_x8_check_remainder @@ -2325,7 +2325,7 @@ vmovdqa xmm5,xmm12 vmovdqa xmm6,xmm13 vmovdqa xmm7,xmm14 - +; move from stack vmovdqa xmm8,XMMWORD[r11] vpxor xmm1,xmm1,XMMWORD[rcx] @@ -2347,7 +2347,7 @@ vaesenc xmm7,xmm7,xmm15 vaesenc xmm8,xmm8,xmm15 - vmovdqa xmm14,XMMWORD[r11] + vmovdqa xmm14,XMMWORD[r11] ; deal with CTR8 vpaddd xmm14,xmm14,XMMWORD[eight] vmovdqa XMMWORD[r11],xmm14 vmovdqu xmm15,XMMWORD[32+rcx] @@ -2488,7 +2488,7 @@ vaesenclast xmm8,xmm8,xmm15 - +; XOR with Plaintext vpxor xmm1,xmm1,XMMWORD[rdi] vpxor xmm2,xmm2,XMMWORD[16+rdi] vpxor xmm3,xmm3,XMMWORD[32+rdi] @@ -2519,8 +2519,8 @@ je NEAR $L$256_enc_msg_x8_out $L$256_enc_msg_x8_loop2: - - +; encrypt each block separately +; CTR1 is the highest counter (even if no LOOP done) vmovdqa xmm1,xmm0 vpaddd xmm0,xmm0,XMMWORD[one] @@ -2540,7 +2540,7 @@ vaesenc xmm1,xmm1,XMMWORD[208+rcx] vaesenclast xmm1,xmm1,XMMWORD[224+rcx] - +; XOR with Plaintext vpxor xmm1,xmm1,XMMWORD[rdi] vmovdqu XMMWORD[rsi],xmm1 @@ -2584,10 +2584,10 @@ $L$256_dec_start: vzeroupper vmovdqa xmm0,XMMWORD[rdx] - - +; The claimed tag is provided after the current calculated tag value. +; CTRBLKs is made from it. vmovdqu xmm15,XMMWORD[16+rdx] - vpor xmm15,xmm15,XMMWORD[OR_MASK] + vpor xmm15,xmm15,XMMWORD[OR_MASK] ; CTR = [1]TAG[126...32][00..00] mov rax,rdx lea rax,[32+rax] @@ -2595,11 +2595,11 @@ and r9,~15 - +; If less then 6 blocks, make singles cmp r9,96 jb NEAR $L$256_dec_loop2 - +; Decrypt the first six blocks sub r9,96 vmovdqa xmm7,xmm15 vpaddd xmm8,xmm7,XMMWORD[one] @@ -2728,7 +2728,7 @@ vaesenclast xmm11,xmm11,xmm4 vaesenclast xmm12,xmm12,xmm4 - +; XOR with CT vpxor xmm7,xmm7,XMMWORD[rdi] vpxor xmm8,xmm8,XMMWORD[16+rdi] vpxor xmm9,xmm9,XMMWORD[32+rdi] @@ -2747,7 +2747,7 @@ add rsi,96 jmp NEAR $L$256_dec_loop1 - +; Decrypt 6 blocks each time while hashing previous 6 blocks ALIGN 64 $L$256_dec_loop1: cmp r9,96 @@ -3091,9 +3091,9 @@ vpxor xmm0,xmm0,xmm5 $L$256_dec_loop2: +; Here we encrypt any remaining whole block - - +; if there are no whole blocks cmp r9,16 jb NEAR $L$256_dec_out sub r9,16 @@ -3149,11 +3149,11 @@ _CET_ENDBR +; parameter 1: %rdi Pointer to NONCE +; parameter 2: %rsi Pointer to CT +; parameter 4: %rdx Pointer to keys - - - - vmovdqa xmm1,XMMWORD[rdx] + vmovdqa xmm1,XMMWORD[rdx] ; xmm1 = first 16 bytes of random key vmovdqa xmm4,XMMWORD[rdi] vmovdqa xmm11,XMMWORD[and_mask] vmovdqa xmm8,XMMWORD[one]
diff --git a/gen/crypto/chacha-x86_64-apple.S b/gen/crypto/chacha-x86_64-apple.S index a6f6c69..7fdeb82 100644 --- a/gen/crypto/chacha-x86_64-apple.S +++ b/gen/crypto/chacha-x86_64-apple.S
@@ -62,25 +62,25 @@ L$ctr32_body: - +// movdqa .Lsigma(%rip),%xmm0 movdqu (%rcx),%xmm1 movdqu 16(%rcx),%xmm2 movdqu (%r8),%xmm3 movdqa L$one(%rip),%xmm4 - - movdqa %xmm1,16(%rsp) - movdqa %xmm2,32(%rsp) - movdqa %xmm3,48(%rsp) - movq %rdx,%rbp +// movdqa %xmm0,4*0(%rsp) # key[0] + movdqa %xmm1,16(%rsp) // key[1] + movdqa %xmm2,32(%rsp) // key[2] + movdqa %xmm3,48(%rsp) // key[3] + movq %rdx,%rbp // reassign %rdx jmp L$oop_outer .p2align 5 L$oop_outer: - movl $0x61707865,%eax - movl $0x3320646e,%ebx - movl $0x79622d32,%ecx - movl $0x6b206574,%edx + movl $0x61707865,%eax // 'expa' + movl $0x3320646e,%ebx // 'nd 3' + movl $0x79622d32,%ecx // '2-by' + movl $0x6b206574,%edx // 'te k' movl 16(%rsp),%r8d movl 20(%rsp),%r9d movl 24(%rsp),%r10d @@ -90,13 +90,13 @@ movl 56(%rsp),%r14d movl 60(%rsp),%r15d - movq %rbp,64+0(%rsp) + movq %rbp,64+0(%rsp) // save len movl $10,%ebp - movq %rsi,64+8(%rsp) - movq %xmm2,%rsi - movq %rdi,64+16(%rsp) + movq %rsi,64+8(%rsp) // save inp + movq %xmm2,%rsi // "%nox" + movq %rdi,64+16(%rsp) // save out movq %rsi,%rdi - shrq $32,%rdi + shrq $32,%rdi // "%nox" jmp L$oop .p2align 5 @@ -207,18 +207,18 @@ roll $7,%r8d decl %ebp jnz L$oop - movl %edi,36(%rsp) + movl %edi,36(%rsp) // modulo-scheduled movl %esi,32(%rsp) - movq 64(%rsp),%rbp + movq 64(%rsp),%rbp // load len movdqa %xmm2,%xmm1 - movq 64+8(%rsp),%rsi - paddd %xmm4,%xmm3 - movq 64+16(%rsp),%rdi + movq 64+8(%rsp),%rsi // load inp + paddd %xmm4,%xmm3 // increment counter + movq 64+16(%rsp),%rdi // load out - addl $0x61707865,%eax - addl $0x3320646e,%ebx - addl $0x79622d32,%ecx - addl $0x6b206574,%edx + addl $0x61707865,%eax // 'expa' + addl $0x3320646e,%ebx // 'nd 3' + addl $0x79622d32,%ecx // '2-by' + addl $0x6b206574,%edx // 'te k' addl 16(%rsp),%r8d addl 20(%rsp),%r9d addl 24(%rsp),%r10d @@ -232,7 +232,7 @@ cmpq $64,%rbp jb L$tail - xorl 0(%rsi),%eax + xorl 0(%rsi),%eax // xor with input xorl 4(%rsi),%ebx xorl 8(%rsi),%ecx xorl 12(%rsi),%edx @@ -245,13 +245,13 @@ xorl 52(%rsi),%r13d xorl 56(%rsi),%r14d xorl 60(%rsi),%r15d - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // inp+=64 pxor %xmm1,%xmm0 movdqa %xmm2,32(%rsp) movd %xmm3,48(%rsp) - movl %eax,0(%rdi) + movl %eax,0(%rdi) // write output movl %ebx,4(%rdi) movl %ecx,8(%rdi) movl %edx,12(%rdi) @@ -264,7 +264,7 @@ movl %r13d,52(%rdi) movl %r14d,56(%rdi) movl %r15d,60(%rdi) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64 subq $64,%rbp jnz L$oop_outer @@ -325,7 +325,7 @@ _ChaCha20_ctr32_ssse3: _CET_ENDBR - movq %rsp,%r9 + movq %rsp,%r9 // frame pointer subq $64+8,%rsp movdqa L$sigma(%rip),%xmm0 @@ -339,7 +339,7 @@ movdqa %xmm1,16(%rsp) movdqa %xmm2,32(%rsp) movdqa %xmm3,48(%rsp) - movq $10,%r8 + movq $10,%r8 // reuse %r8 jmp L$oop_ssse3 .p2align 5 @@ -410,19 +410,19 @@ movdqu 0(%rsi),%xmm4 movdqu 16(%rsi),%xmm5 - pxor %xmm4,%xmm0 + pxor %xmm4,%xmm0 // xor with input movdqu 32(%rsi),%xmm4 pxor %xmm5,%xmm1 movdqu 48(%rsi),%xmm5 - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // inp+=64 pxor %xmm4,%xmm2 pxor %xmm5,%xmm3 - movdqu %xmm0,0(%rdi) + movdqu %xmm0,0(%rdi) // write output movdqu %xmm1,16(%rdi) movdqu %xmm2,32(%rdi) movdqu %xmm3,48(%rdi) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64 subq $64,%rdx jnz L$oop_outer_ssse3 @@ -461,20 +461,20 @@ _ChaCha20_ctr32_ssse3_4x: _CET_ENDBR - movq %rsp,%r9 + movq %rsp,%r9 // frame pointer subq $0x140+8,%rsp - movdqa L$sigma(%rip),%xmm11 - movdqu (%rcx),%xmm15 - movdqu 16(%rcx),%xmm7 - movdqu (%r8),%xmm3 - leaq 256(%rsp),%rcx + movdqa L$sigma(%rip),%xmm11 // key[0] + movdqu (%rcx),%xmm15 // key[1] + movdqu 16(%rcx),%xmm7 // key[2] + movdqu (%r8),%xmm3 // key[3] + leaq 256(%rsp),%rcx // size optimization leaq L$rot16(%rip),%r10 leaq L$rot24(%rip),%r11 - pshufd $0x00,%xmm11,%xmm8 + pshufd $0x00,%xmm11,%xmm8 // smash key by lanes... pshufd $0x55,%xmm11,%xmm9 - movdqa %xmm8,64(%rsp) + movdqa %xmm8,64(%rsp) // ... and offload pshufd $0xaa,%xmm11,%xmm10 movdqa %xmm9,80(%rsp) pshufd $0xff,%xmm11,%xmm11 @@ -490,18 +490,18 @@ movdqa %xmm14,160-256(%rcx) movdqa %xmm15,176-256(%rcx) - pshufd $0x00,%xmm7,%xmm4 - pshufd $0x55,%xmm7,%xmm5 + pshufd $0x00,%xmm7,%xmm4 // "" + pshufd $0x55,%xmm7,%xmm5 // "" movdqa %xmm4,192-256(%rcx) - pshufd $0xaa,%xmm7,%xmm6 + pshufd $0xaa,%xmm7,%xmm6 // "" movdqa %xmm5,208-256(%rcx) - pshufd $0xff,%xmm7,%xmm7 + pshufd $0xff,%xmm7,%xmm7 // "" movdqa %xmm6,224-256(%rcx) movdqa %xmm7,240-256(%rcx) pshufd $0x00,%xmm3,%xmm0 pshufd $0x55,%xmm3,%xmm1 - paddd L$inc(%rip),%xmm0 + paddd L$inc(%rip),%xmm0 // don't save counters yet pshufd $0xaa,%xmm3,%xmm2 movdqa %xmm1,272-256(%rcx) pshufd $0xff,%xmm3,%xmm3 @@ -512,7 +512,7 @@ .p2align 5 L$oop_outer4x: - movdqa 64(%rsp),%xmm8 + movdqa 64(%rsp),%xmm8 // re-load smashed key movdqa 80(%rsp),%xmm9 movdqa 96(%rsp),%xmm10 movdqa 112(%rsp),%xmm11 @@ -520,22 +520,22 @@ movdqa 144-256(%rcx),%xmm13 movdqa 160-256(%rcx),%xmm14 movdqa 176-256(%rcx),%xmm15 - movdqa 192-256(%rcx),%xmm4 - movdqa 208-256(%rcx),%xmm5 - movdqa 224-256(%rcx),%xmm6 - movdqa 240-256(%rcx),%xmm7 + movdqa 192-256(%rcx),%xmm4 // "" + movdqa 208-256(%rcx),%xmm5 // "" + movdqa 224-256(%rcx),%xmm6 // "" + movdqa 240-256(%rcx),%xmm7 // "" movdqa 256-256(%rcx),%xmm0 movdqa 272-256(%rcx),%xmm1 movdqa 288-256(%rcx),%xmm2 movdqa 304-256(%rcx),%xmm3 - paddd L$four(%rip),%xmm0 + paddd L$four(%rip),%xmm0 // next SIMD counters L$oop_enter4x: - movdqa %xmm6,32(%rsp) - movdqa %xmm7,48(%rsp) - movdqa (%r10),%xmm7 + movdqa %xmm6,32(%rsp) // SIMD equivalent of "%nox" + movdqa %xmm7,48(%rsp) // SIMD equivalent of "%nox" + movdqa (%r10),%xmm7 // .Lrot16(%rip) movl $10,%eax - movdqa %xmm0,256-256(%rcx) + movdqa %xmm0,256-256(%rcx) // save SIMD counters jmp L$oop4x .p2align 5 @@ -703,32 +703,32 @@ decl %eax jnz L$oop4x - paddd 64(%rsp),%xmm8 + paddd 64(%rsp),%xmm8 // accumulate key material paddd 80(%rsp),%xmm9 paddd 96(%rsp),%xmm10 paddd 112(%rsp),%xmm11 - movdqa %xmm8,%xmm6 + movdqa %xmm8,%xmm6 // "de-interlace" data punpckldq %xmm9,%xmm8 movdqa %xmm10,%xmm7 punpckldq %xmm11,%xmm10 punpckhdq %xmm9,%xmm6 punpckhdq %xmm11,%xmm7 movdqa %xmm8,%xmm9 - punpcklqdq %xmm10,%xmm8 + punpcklqdq %xmm10,%xmm8 // "a0" movdqa %xmm6,%xmm11 - punpcklqdq %xmm7,%xmm6 - punpckhqdq %xmm10,%xmm9 - punpckhqdq %xmm7,%xmm11 + punpcklqdq %xmm7,%xmm6 // "a2" + punpckhqdq %xmm10,%xmm9 // "a1" + punpckhqdq %xmm7,%xmm11 // "a3" paddd 128-256(%rcx),%xmm12 paddd 144-256(%rcx),%xmm13 paddd 160-256(%rcx),%xmm14 paddd 176-256(%rcx),%xmm15 - movdqa %xmm8,0(%rsp) + movdqa %xmm8,0(%rsp) // offload movdqa %xmm9,16(%rsp) - movdqa 32(%rsp),%xmm8 - movdqa 48(%rsp),%xmm9 + movdqa 32(%rsp),%xmm8 // "xc2" + movdqa 48(%rsp),%xmm9 // "xc3" movdqa %xmm12,%xmm10 punpckldq %xmm13,%xmm12 @@ -737,17 +737,17 @@ punpckhdq %xmm13,%xmm10 punpckhdq %xmm15,%xmm7 movdqa %xmm12,%xmm13 - punpcklqdq %xmm14,%xmm12 + punpcklqdq %xmm14,%xmm12 // "b0" movdqa %xmm10,%xmm15 - punpcklqdq %xmm7,%xmm10 - punpckhqdq %xmm14,%xmm13 - punpckhqdq %xmm7,%xmm15 + punpcklqdq %xmm7,%xmm10 // "b2" + punpckhqdq %xmm14,%xmm13 // "b1" + punpckhqdq %xmm7,%xmm15 // "b3" paddd 192-256(%rcx),%xmm4 paddd 208-256(%rcx),%xmm5 paddd 224-256(%rcx),%xmm8 paddd 240-256(%rcx),%xmm9 - movdqa %xmm6,32(%rsp) + movdqa %xmm6,32(%rsp) // keep offloading movdqa %xmm11,48(%rsp) movdqa %xmm4,%xmm14 @@ -757,11 +757,11 @@ punpckhdq %xmm5,%xmm14 punpckhdq %xmm9,%xmm7 movdqa %xmm4,%xmm5 - punpcklqdq %xmm8,%xmm4 + punpcklqdq %xmm8,%xmm4 // "c0" movdqa %xmm14,%xmm9 - punpcklqdq %xmm7,%xmm14 - punpckhqdq %xmm8,%xmm5 - punpckhqdq %xmm7,%xmm9 + punpcklqdq %xmm7,%xmm14 // "c2" + punpckhqdq %xmm8,%xmm5 // "c1" + punpckhqdq %xmm7,%xmm9 // "c3" paddd 256-256(%rcx),%xmm0 paddd 272-256(%rcx),%xmm1 paddd 288-256(%rcx),%xmm2 @@ -774,19 +774,19 @@ punpckhdq %xmm1,%xmm8 punpckhdq %xmm3,%xmm7 movdqa %xmm0,%xmm1 - punpcklqdq %xmm2,%xmm0 + punpcklqdq %xmm2,%xmm0 // "d0" movdqa %xmm8,%xmm3 - punpcklqdq %xmm7,%xmm8 - punpckhqdq %xmm2,%xmm1 - punpckhqdq %xmm7,%xmm3 + punpcklqdq %xmm7,%xmm8 // "d2" + punpckhqdq %xmm2,%xmm1 // "d1" + punpckhqdq %xmm7,%xmm3 // "d3" cmpq $256,%rdx jb L$tail4x - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -799,7 +799,7 @@ movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 @@ -812,7 +812,7 @@ movdqu %xmm2,96(%rdi) movdqu 32(%rsi),%xmm2 movdqu %xmm7,112(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization movdqu 48(%rsi),%xmm7 pxor 32(%rsp),%xmm6 pxor %xmm10,%xmm11 @@ -827,7 +827,7 @@ movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // inp+=64*4 pxor 48(%rsp),%xmm6 pxor %xmm15,%xmm11 pxor %xmm9,%xmm2 @@ -836,7 +836,7 @@ movdqu %xmm11,80(%rdi) movdqu %xmm2,96(%rdi) movdqu %xmm7,112(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // out+=64*4 subq $256,%rdx jnz L$oop_outer4x @@ -851,9 +851,9 @@ cmpq $64,%rdx jae L$64_or_more4x - +// movdqa 0x00(%rsp),%xmm6 # is offloaded, remember? xorq %r10,%r10 - +// movdqa %xmm6,0x00(%rsp) movdqa %xmm12,16(%rsp) movdqa %xmm4,32(%rsp) movdqa %xmm0,48(%rsp) @@ -861,11 +861,11 @@ .p2align 5 L$64_or_more4x: - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -875,24 +875,24 @@ movdqu %xmm7,48(%rdi) je L$done4x - movdqa 16(%rsp),%xmm6 - leaq 64(%rsi),%rsi + movdqa 16(%rsp),%xmm6 // is offloaded, remember? + leaq 64(%rsi),%rsi // inp+=64*1 xorq %r10,%r10 movdqa %xmm6,0(%rsp) movdqa %xmm13,16(%rsp) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64*1 movdqa %xmm5,32(%rsp) - subq $64,%rdx + subq $64,%rdx // len-=64*1 movdqa %xmm1,48(%rsp) jmp L$oop_tail4x .p2align 5 L$128_or_more4x: - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -915,24 +915,24 @@ movdqu %xmm7,112(%rdi) je L$done4x - movdqa 32(%rsp),%xmm6 - leaq 128(%rsi),%rsi + movdqa 32(%rsp),%xmm6 // is offloaded, remember? + leaq 128(%rsi),%rsi // inp+=64*2 xorq %r10,%r10 movdqa %xmm6,0(%rsp) movdqa %xmm10,16(%rsp) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // out+=64*2 movdqa %xmm14,32(%rsp) - subq $128,%rdx + subq $128,%rdx // len-=64*2 movdqa %xmm8,48(%rsp) jmp L$oop_tail4x .p2align 5 L$192_or_more4x: - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -945,7 +945,7 @@ movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 @@ -958,7 +958,7 @@ movdqu %xmm2,96(%rdi) movdqu 32(%rsi),%xmm2 movdqu %xmm7,112(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization movdqu 48(%rsi),%xmm7 pxor 32(%rsp),%xmm6 pxor %xmm10,%xmm11 @@ -970,14 +970,14 @@ movdqu %xmm7,48(%rdi) je L$done4x - movdqa 48(%rsp),%xmm6 - leaq 64(%rsi),%rsi + movdqa 48(%rsp),%xmm6 // is offloaded, remember? + leaq 64(%rsi),%rsi // inp+=64*3 xorq %r10,%r10 movdqa %xmm6,0(%rsp) movdqa %xmm15,16(%rsp) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64*3 movdqa %xmm9,32(%rsp) - subq $192,%rdx + subq $192,%rdx // len-=64*3 movdqa %xmm3,48(%rsp) L$oop_tail4x: @@ -1004,33 +1004,33 @@ _ChaCha20_ctr32_avx2: _CET_ENDBR - movq %rsp,%r9 + movq %rsp,%r9 // frame register subq $0x280+8,%rsp andq $-32,%rsp vzeroupper +// ############### stack layout +// +0x00 SIMD equivalent of %r12d +// ... +// +0x80 constant copy of key[0-2] smashed by lanes +// ... +// +0x200 SIMD counters (with nonce smashed by lanes) +// ... +// +0x280 - - - - - - - - - vbroadcasti128 L$sigma(%rip),%ymm11 - vbroadcasti128 (%rcx),%ymm3 - vbroadcasti128 16(%rcx),%ymm15 - vbroadcasti128 (%r8),%ymm7 - leaq 256(%rsp),%rcx - leaq 512(%rsp),%rax + vbroadcasti128 L$sigma(%rip),%ymm11 // key[0] + vbroadcasti128 (%rcx),%ymm3 // key[1] + vbroadcasti128 16(%rcx),%ymm15 // key[2] + vbroadcasti128 (%r8),%ymm7 // key[3] + leaq 256(%rsp),%rcx // size optimization + leaq 512(%rsp),%rax // size optimization leaq L$rot16(%rip),%r10 leaq L$rot24(%rip),%r11 - vpshufd $0x00,%ymm11,%ymm8 + vpshufd $0x00,%ymm11,%ymm8 // smash key by lanes... vpshufd $0x55,%ymm11,%ymm9 - vmovdqa %ymm8,128-256(%rcx) + vmovdqa %ymm8,128-256(%rcx) // ... and offload vpshufd $0xaa,%ymm11,%ymm10 vmovdqa %ymm9,160-256(%rcx) vpshufd $0xff,%ymm11,%ymm11 @@ -1046,18 +1046,18 @@ vmovdqa %ymm2,320-256(%rcx) vmovdqa %ymm3,352-256(%rcx) - vpshufd $0x00,%ymm15,%ymm12 - vpshufd $0x55,%ymm15,%ymm13 + vpshufd $0x00,%ymm15,%ymm12 // "xc0" + vpshufd $0x55,%ymm15,%ymm13 // "xc1" vmovdqa %ymm12,384-512(%rax) - vpshufd $0xaa,%ymm15,%ymm14 + vpshufd $0xaa,%ymm15,%ymm14 // "xc2" vmovdqa %ymm13,416-512(%rax) - vpshufd $0xff,%ymm15,%ymm15 + vpshufd $0xff,%ymm15,%ymm15 // "xc3" vmovdqa %ymm14,448-512(%rax) vmovdqa %ymm15,480-512(%rax) vpshufd $0x00,%ymm7,%ymm4 vpshufd $0x55,%ymm7,%ymm5 - vpaddd L$incy(%rip),%ymm4,%ymm4 + vpaddd L$incy(%rip),%ymm4,%ymm4 // don't save counters yet vpshufd $0xaa,%ymm7,%ymm6 vmovdqa %ymm5,544-512(%rax) vpshufd $0xff,%ymm7,%ymm7 @@ -1068,7 +1068,7 @@ .p2align 5 L$oop_outer8x: - vmovdqa 128-256(%rcx),%ymm8 + vmovdqa 128-256(%rcx),%ymm8 // re-load smashed key vmovdqa 160-256(%rcx),%ymm9 vmovdqa 192-256(%rcx),%ymm10 vmovdqa 224-256(%rcx),%ymm11 @@ -1076,21 +1076,21 @@ vmovdqa 288-256(%rcx),%ymm1 vmovdqa 320-256(%rcx),%ymm2 vmovdqa 352-256(%rcx),%ymm3 - vmovdqa 384-512(%rax),%ymm12 - vmovdqa 416-512(%rax),%ymm13 - vmovdqa 448-512(%rax),%ymm14 - vmovdqa 480-512(%rax),%ymm15 + vmovdqa 384-512(%rax),%ymm12 // "xc0" + vmovdqa 416-512(%rax),%ymm13 // "xc1" + vmovdqa 448-512(%rax),%ymm14 // "xc2" + vmovdqa 480-512(%rax),%ymm15 // "xc3" vmovdqa 512-512(%rax),%ymm4 vmovdqa 544-512(%rax),%ymm5 vmovdqa 576-512(%rax),%ymm6 vmovdqa 608-512(%rax),%ymm7 - vpaddd L$eight(%rip),%ymm4,%ymm4 + vpaddd L$eight(%rip),%ymm4,%ymm4 // next SIMD counters L$oop_enter8x: - vmovdqa %ymm14,64(%rsp) - vmovdqa %ymm15,96(%rsp) + vmovdqa %ymm14,64(%rsp) // SIMD equivalent of "%nox" + vmovdqa %ymm15,96(%rsp) // SIMD equivalent of "%nox" vbroadcasti128 (%r10),%ymm15 - vmovdqa %ymm4,512-512(%rax) + vmovdqa %ymm4,512-512(%rax) // save SIMD counters movl $10,%eax jmp L$oop8x @@ -1243,20 +1243,20 @@ decl %eax jnz L$oop8x - leaq 512(%rsp),%rax - vpaddd 128-256(%rcx),%ymm8,%ymm8 + leaq 512(%rsp),%rax // size optimization + vpaddd 128-256(%rcx),%ymm8,%ymm8 // accumulate key vpaddd 160-256(%rcx),%ymm9,%ymm9 vpaddd 192-256(%rcx),%ymm10,%ymm10 vpaddd 224-256(%rcx),%ymm11,%ymm11 - vpunpckldq %ymm9,%ymm8,%ymm14 + vpunpckldq %ymm9,%ymm8,%ymm14 // "de-interlace" data vpunpckldq %ymm11,%ymm10,%ymm15 vpunpckhdq %ymm9,%ymm8,%ymm8 vpunpckhdq %ymm11,%ymm10,%ymm10 - vpunpcklqdq %ymm15,%ymm14,%ymm9 - vpunpckhqdq %ymm15,%ymm14,%ymm14 - vpunpcklqdq %ymm10,%ymm8,%ymm11 - vpunpckhqdq %ymm10,%ymm8,%ymm8 + vpunpcklqdq %ymm15,%ymm14,%ymm9 // "a0" + vpunpckhqdq %ymm15,%ymm14,%ymm14 // "a1" + vpunpcklqdq %ymm10,%ymm8,%ymm11 // "a2" + vpunpckhqdq %ymm10,%ymm8,%ymm8 // "a3" vpaddd 256-256(%rcx),%ymm0,%ymm0 vpaddd 288-256(%rcx),%ymm1,%ymm1 vpaddd 320-256(%rcx),%ymm2,%ymm2 @@ -1266,11 +1266,11 @@ vpunpckldq %ymm3,%ymm2,%ymm15 vpunpckhdq %ymm1,%ymm0,%ymm0 vpunpckhdq %ymm3,%ymm2,%ymm2 - vpunpcklqdq %ymm15,%ymm10,%ymm1 - vpunpckhqdq %ymm15,%ymm10,%ymm10 - vpunpcklqdq %ymm2,%ymm0,%ymm3 - vpunpckhqdq %ymm2,%ymm0,%ymm0 - vperm2i128 $0x20,%ymm1,%ymm9,%ymm15 + vpunpcklqdq %ymm15,%ymm10,%ymm1 // "b0" + vpunpckhqdq %ymm15,%ymm10,%ymm10 // "b1" + vpunpcklqdq %ymm2,%ymm0,%ymm3 // "b2" + vpunpckhqdq %ymm2,%ymm0,%ymm0 // "b3" + vperm2i128 $0x20,%ymm1,%ymm9,%ymm15 // "de-interlace" further vperm2i128 $0x31,%ymm1,%ymm9,%ymm1 vperm2i128 $0x20,%ymm10,%ymm14,%ymm9 vperm2i128 $0x31,%ymm10,%ymm14,%ymm10 @@ -1278,10 +1278,10 @@ vperm2i128 $0x31,%ymm3,%ymm11,%ymm3 vperm2i128 $0x20,%ymm0,%ymm8,%ymm11 vperm2i128 $0x31,%ymm0,%ymm8,%ymm0 - vmovdqa %ymm15,0(%rsp) + vmovdqa %ymm15,0(%rsp) // offload vmovdqa %ymm9,32(%rsp) - vmovdqa 64(%rsp),%ymm15 - vmovdqa 96(%rsp),%ymm9 + vmovdqa 64(%rsp),%ymm15 // %ymm15 + vmovdqa 96(%rsp),%ymm9 // %ymm9 vpaddd 384-512(%rax),%ymm12,%ymm12 vpaddd 416-512(%rax),%ymm13,%ymm13 @@ -1292,10 +1292,10 @@ vpunpckldq %ymm9,%ymm15,%ymm8 vpunpckhdq %ymm13,%ymm12,%ymm12 vpunpckhdq %ymm9,%ymm15,%ymm15 - vpunpcklqdq %ymm8,%ymm2,%ymm13 - vpunpckhqdq %ymm8,%ymm2,%ymm2 - vpunpcklqdq %ymm15,%ymm12,%ymm9 - vpunpckhqdq %ymm15,%ymm12,%ymm12 + vpunpcklqdq %ymm8,%ymm2,%ymm13 // "c0" + vpunpckhqdq %ymm8,%ymm2,%ymm2 // "c1" + vpunpcklqdq %ymm15,%ymm12,%ymm9 // "c2" + vpunpckhqdq %ymm15,%ymm12,%ymm12 // "c3" vpaddd 512-512(%rax),%ymm4,%ymm4 vpaddd 544-512(%rax),%ymm5,%ymm5 vpaddd 576-512(%rax),%ymm6,%ymm6 @@ -1305,11 +1305,11 @@ vpunpckldq %ymm7,%ymm6,%ymm8 vpunpckhdq %ymm5,%ymm4,%ymm4 vpunpckhdq %ymm7,%ymm6,%ymm6 - vpunpcklqdq %ymm8,%ymm15,%ymm5 - vpunpckhqdq %ymm8,%ymm15,%ymm15 - vpunpcklqdq %ymm6,%ymm4,%ymm7 - vpunpckhqdq %ymm6,%ymm4,%ymm4 - vperm2i128 $0x20,%ymm5,%ymm13,%ymm8 + vpunpcklqdq %ymm8,%ymm15,%ymm5 // "d0" + vpunpckhqdq %ymm8,%ymm15,%ymm15 // "d1" + vpunpcklqdq %ymm6,%ymm4,%ymm7 // "d2" + vpunpckhqdq %ymm6,%ymm4,%ymm4 // "d3" + vperm2i128 $0x20,%ymm5,%ymm13,%ymm8 // "de-interlace" further vperm2i128 $0x31,%ymm5,%ymm13,%ymm5 vperm2i128 $0x20,%ymm15,%ymm2,%ymm13 vperm2i128 $0x31,%ymm15,%ymm2,%ymm15 @@ -1317,55 +1317,55 @@ vperm2i128 $0x31,%ymm7,%ymm9,%ymm7 vperm2i128 $0x20,%ymm4,%ymm12,%ymm9 vperm2i128 $0x31,%ymm4,%ymm12,%ymm4 - vmovdqa 0(%rsp),%ymm6 + vmovdqa 0(%rsp),%ymm6 // was offloaded, remember? vmovdqa 32(%rsp),%ymm12 cmpq $512,%rdx jb L$tail8x - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization vpxor 0(%rsi),%ymm12,%ymm12 vpxor 32(%rsi),%ymm13,%ymm13 vpxor 64(%rsi),%ymm10,%ymm10 vpxor 96(%rsi),%ymm15,%ymm15 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm12,0(%rdi) vmovdqu %ymm13,32(%rdi) vmovdqu %ymm10,64(%rdi) vmovdqu %ymm15,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization vpxor 0(%rsi),%ymm14,%ymm14 vpxor 32(%rsi),%ymm2,%ymm2 vpxor 64(%rsi),%ymm3,%ymm3 vpxor 96(%rsi),%ymm7,%ymm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm14,0(%rdi) vmovdqu %ymm2,32(%rdi) vmovdqu %ymm3,64(%rdi) vmovdqu %ymm7,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization vpxor 0(%rsi),%ymm11,%ymm11 vpxor 32(%rsi),%ymm9,%ymm9 vpxor 64(%rsi),%ymm0,%ymm0 vpxor 96(%rsi),%ymm4,%ymm4 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm11,0(%rdi) vmovdqu %ymm9,32(%rdi) vmovdqu %ymm0,64(%rdi) vmovdqu %ymm4,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization subq $512,%rdx jnz L$oop_outer8x @@ -1395,23 +1395,23 @@ .p2align 5 L$64_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) je L$done8x - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // inp+=64*1 xorq %r10,%r10 vmovdqa %ymm1,0(%rsp) - leaq 64(%rdi),%rdi - subq $64,%rdx + leaq 64(%rdi),%rdi // out+=64*1 + subq $64,%rdx // len-=64*1 vmovdqa %ymm5,32(%rsp) jmp L$oop_tail8x .p2align 5 L$128_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1421,17 +1421,17 @@ vmovdqu %ymm5,96(%rdi) je L$done8x - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // inp+=64*2 xorq %r10,%r10 vmovdqa %ymm12,0(%rsp) - leaq 128(%rdi),%rdi - subq $128,%rdx + leaq 128(%rdi),%rdi // out+=64*2 + subq $128,%rdx // len-=64*2 vmovdqa %ymm13,32(%rsp) jmp L$oop_tail8x .p2align 5 L$192_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1445,17 +1445,17 @@ vmovdqu %ymm13,160(%rdi) je L$done8x - leaq 192(%rsi),%rsi + leaq 192(%rsi),%rsi // inp+=64*3 xorq %r10,%r10 vmovdqa %ymm10,0(%rsp) - leaq 192(%rdi),%rdi - subq $192,%rdx + leaq 192(%rdi),%rdi // out+=64*3 + subq $192,%rdx // len-=64*3 vmovdqa %ymm15,32(%rsp) jmp L$oop_tail8x .p2align 5 L$256_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1473,17 +1473,17 @@ vmovdqu %ymm15,224(%rdi) je L$done8x - leaq 256(%rsi),%rsi + leaq 256(%rsi),%rsi // inp+=64*4 xorq %r10,%r10 vmovdqa %ymm14,0(%rsp) - leaq 256(%rdi),%rdi - subq $256,%rdx + leaq 256(%rdi),%rdi // out+=64*4 + subq $256,%rdx // len-=64*4 vmovdqa %ymm2,32(%rsp) jmp L$oop_tail8x .p2align 5 L$320_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1505,17 +1505,17 @@ vmovdqu %ymm2,288(%rdi) je L$done8x - leaq 320(%rsi),%rsi + leaq 320(%rsi),%rsi // inp+=64*5 xorq %r10,%r10 vmovdqa %ymm3,0(%rsp) - leaq 320(%rdi),%rdi - subq $320,%rdx + leaq 320(%rdi),%rdi // out+=64*5 + subq $320,%rdx // len-=64*5 vmovdqa %ymm7,32(%rsp) jmp L$oop_tail8x .p2align 5 L$384_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1541,17 +1541,17 @@ vmovdqu %ymm7,352(%rdi) je L$done8x - leaq 384(%rsi),%rsi + leaq 384(%rsi),%rsi // inp+=64*6 xorq %r10,%r10 vmovdqa %ymm11,0(%rsp) - leaq 384(%rdi),%rdi - subq $384,%rdx + leaq 384(%rdi),%rdi // out+=64*6 + subq $384,%rdx // len-=64*6 vmovdqa %ymm9,32(%rsp) jmp L$oop_tail8x .p2align 5 L$448_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1581,11 +1581,11 @@ vmovdqu %ymm9,416(%rdi) je L$done8x - leaq 448(%rsi),%rsi + leaq 448(%rsi),%rsi // inp+=64*7 xorq %r10,%r10 vmovdqa %ymm0,0(%rsp) - leaq 448(%rdi),%rdi - subq $448,%rdx + leaq 448(%rdi),%rdi // out+=64*7 + subq $448,%rdx // len-=64*7 vmovdqa %ymm4,32(%rsp) L$oop_tail8x:
diff --git a/gen/crypto/chacha-x86_64-linux.S b/gen/crypto/chacha-x86_64-linux.S index eec46bc..b7ac9eb 100644 --- a/gen/crypto/chacha-x86_64-linux.S +++ b/gen/crypto/chacha-x86_64-linux.S
@@ -68,25 +68,25 @@ .cfi_adjust_cfa_offset 88 .Lctr32_body: - +// movdqa .Lsigma(%rip),%xmm0 movdqu (%rcx),%xmm1 movdqu 16(%rcx),%xmm2 movdqu (%r8),%xmm3 movdqa .Lone(%rip),%xmm4 - - movdqa %xmm1,16(%rsp) - movdqa %xmm2,32(%rsp) - movdqa %xmm3,48(%rsp) - movq %rdx,%rbp +// movdqa %xmm0,4*0(%rsp) # key[0] + movdqa %xmm1,16(%rsp) // key[1] + movdqa %xmm2,32(%rsp) // key[2] + movdqa %xmm3,48(%rsp) // key[3] + movq %rdx,%rbp // reassign %rdx jmp .Loop_outer .align 32 .Loop_outer: - movl $0x61707865,%eax - movl $0x3320646e,%ebx - movl $0x79622d32,%ecx - movl $0x6b206574,%edx + movl $0x61707865,%eax // 'expa' + movl $0x3320646e,%ebx // 'nd 3' + movl $0x79622d32,%ecx // '2-by' + movl $0x6b206574,%edx // 'te k' movl 16(%rsp),%r8d movl 20(%rsp),%r9d movl 24(%rsp),%r10d @@ -96,13 +96,13 @@ movl 56(%rsp),%r14d movl 60(%rsp),%r15d - movq %rbp,64+0(%rsp) + movq %rbp,64+0(%rsp) // save len movl $10,%ebp - movq %rsi,64+8(%rsp) - movq %xmm2,%rsi - movq %rdi,64+16(%rsp) + movq %rsi,64+8(%rsp) // save inp + movq %xmm2,%rsi // "%nox" + movq %rdi,64+16(%rsp) // save out movq %rsi,%rdi - shrq $32,%rdi + shrq $32,%rdi // "%nox" jmp .Loop .align 32 @@ -213,18 +213,18 @@ roll $7,%r8d decl %ebp jnz .Loop - movl %edi,36(%rsp) + movl %edi,36(%rsp) // modulo-scheduled movl %esi,32(%rsp) - movq 64(%rsp),%rbp + movq 64(%rsp),%rbp // load len movdqa %xmm2,%xmm1 - movq 64+8(%rsp),%rsi - paddd %xmm4,%xmm3 - movq 64+16(%rsp),%rdi + movq 64+8(%rsp),%rsi // load inp + paddd %xmm4,%xmm3 // increment counter + movq 64+16(%rsp),%rdi // load out - addl $0x61707865,%eax - addl $0x3320646e,%ebx - addl $0x79622d32,%ecx - addl $0x6b206574,%edx + addl $0x61707865,%eax // 'expa' + addl $0x3320646e,%ebx // 'nd 3' + addl $0x79622d32,%ecx // '2-by' + addl $0x6b206574,%edx // 'te k' addl 16(%rsp),%r8d addl 20(%rsp),%r9d addl 24(%rsp),%r10d @@ -238,7 +238,7 @@ cmpq $64,%rbp jb .Ltail - xorl 0(%rsi),%eax + xorl 0(%rsi),%eax // xor with input xorl 4(%rsi),%ebx xorl 8(%rsi),%ecx xorl 12(%rsi),%edx @@ -251,13 +251,13 @@ xorl 52(%rsi),%r13d xorl 56(%rsi),%r14d xorl 60(%rsi),%r15d - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // inp+=64 pxor %xmm1,%xmm0 movdqa %xmm2,32(%rsp) movd %xmm3,48(%rsp) - movl %eax,0(%rdi) + movl %eax,0(%rdi) // write output movl %ebx,4(%rdi) movl %ecx,8(%rdi) movl %edx,12(%rdi) @@ -270,7 +270,7 @@ movl %r13d,52(%rdi) movl %r14d,56(%rdi) movl %r15d,60(%rdi) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64 subq $64,%rbp jnz .Loop_outer @@ -330,7 +330,7 @@ ChaCha20_ctr32_ssse3: .cfi_startproc _CET_ENDBR - movq %rsp,%r9 + movq %rsp,%r9 // frame pointer .cfi_def_cfa_register r9 subq $64+8,%rsp movdqa .Lsigma(%rip),%xmm0 @@ -344,7 +344,7 @@ movdqa %xmm1,16(%rsp) movdqa %xmm2,32(%rsp) movdqa %xmm3,48(%rsp) - movq $10,%r8 + movq $10,%r8 // reuse %r8 jmp .Loop_ssse3 .align 32 @@ -415,19 +415,19 @@ movdqu 0(%rsi),%xmm4 movdqu 16(%rsi),%xmm5 - pxor %xmm4,%xmm0 + pxor %xmm4,%xmm0 // xor with input movdqu 32(%rsi),%xmm4 pxor %xmm5,%xmm1 movdqu 48(%rsi),%xmm5 - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // inp+=64 pxor %xmm4,%xmm2 pxor %xmm5,%xmm3 - movdqu %xmm0,0(%rdi) + movdqu %xmm0,0(%rdi) // write output movdqu %xmm1,16(%rdi) movdqu %xmm2,32(%rdi) movdqu %xmm3,48(%rdi) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64 subq $64,%rdx jnz .Loop_outer_ssse3 @@ -465,20 +465,20 @@ ChaCha20_ctr32_ssse3_4x: .cfi_startproc _CET_ENDBR - movq %rsp,%r9 + movq %rsp,%r9 // frame pointer .cfi_def_cfa_register r9 subq $0x140+8,%rsp - movdqa .Lsigma(%rip),%xmm11 - movdqu (%rcx),%xmm15 - movdqu 16(%rcx),%xmm7 - movdqu (%r8),%xmm3 - leaq 256(%rsp),%rcx + movdqa .Lsigma(%rip),%xmm11 // key[0] + movdqu (%rcx),%xmm15 // key[1] + movdqu 16(%rcx),%xmm7 // key[2] + movdqu (%r8),%xmm3 // key[3] + leaq 256(%rsp),%rcx // size optimization leaq .Lrot16(%rip),%r10 leaq .Lrot24(%rip),%r11 - pshufd $0x00,%xmm11,%xmm8 + pshufd $0x00,%xmm11,%xmm8 // smash key by lanes... pshufd $0x55,%xmm11,%xmm9 - movdqa %xmm8,64(%rsp) + movdqa %xmm8,64(%rsp) // ... and offload pshufd $0xaa,%xmm11,%xmm10 movdqa %xmm9,80(%rsp) pshufd $0xff,%xmm11,%xmm11 @@ -494,18 +494,18 @@ movdqa %xmm14,160-256(%rcx) movdqa %xmm15,176-256(%rcx) - pshufd $0x00,%xmm7,%xmm4 - pshufd $0x55,%xmm7,%xmm5 + pshufd $0x00,%xmm7,%xmm4 // "" + pshufd $0x55,%xmm7,%xmm5 // "" movdqa %xmm4,192-256(%rcx) - pshufd $0xaa,%xmm7,%xmm6 + pshufd $0xaa,%xmm7,%xmm6 // "" movdqa %xmm5,208-256(%rcx) - pshufd $0xff,%xmm7,%xmm7 + pshufd $0xff,%xmm7,%xmm7 // "" movdqa %xmm6,224-256(%rcx) movdqa %xmm7,240-256(%rcx) pshufd $0x00,%xmm3,%xmm0 pshufd $0x55,%xmm3,%xmm1 - paddd .Linc(%rip),%xmm0 + paddd .Linc(%rip),%xmm0 // don't save counters yet pshufd $0xaa,%xmm3,%xmm2 movdqa %xmm1,272-256(%rcx) pshufd $0xff,%xmm3,%xmm3 @@ -516,7 +516,7 @@ .align 32 .Loop_outer4x: - movdqa 64(%rsp),%xmm8 + movdqa 64(%rsp),%xmm8 // re-load smashed key movdqa 80(%rsp),%xmm9 movdqa 96(%rsp),%xmm10 movdqa 112(%rsp),%xmm11 @@ -524,22 +524,22 @@ movdqa 144-256(%rcx),%xmm13 movdqa 160-256(%rcx),%xmm14 movdqa 176-256(%rcx),%xmm15 - movdqa 192-256(%rcx),%xmm4 - movdqa 208-256(%rcx),%xmm5 - movdqa 224-256(%rcx),%xmm6 - movdqa 240-256(%rcx),%xmm7 + movdqa 192-256(%rcx),%xmm4 // "" + movdqa 208-256(%rcx),%xmm5 // "" + movdqa 224-256(%rcx),%xmm6 // "" + movdqa 240-256(%rcx),%xmm7 // "" movdqa 256-256(%rcx),%xmm0 movdqa 272-256(%rcx),%xmm1 movdqa 288-256(%rcx),%xmm2 movdqa 304-256(%rcx),%xmm3 - paddd .Lfour(%rip),%xmm0 + paddd .Lfour(%rip),%xmm0 // next SIMD counters .Loop_enter4x: - movdqa %xmm6,32(%rsp) - movdqa %xmm7,48(%rsp) - movdqa (%r10),%xmm7 + movdqa %xmm6,32(%rsp) // SIMD equivalent of "%nox" + movdqa %xmm7,48(%rsp) // SIMD equivalent of "%nox" + movdqa (%r10),%xmm7 // .Lrot16(%rip) movl $10,%eax - movdqa %xmm0,256-256(%rcx) + movdqa %xmm0,256-256(%rcx) // save SIMD counters jmp .Loop4x .align 32 @@ -707,32 +707,32 @@ decl %eax jnz .Loop4x - paddd 64(%rsp),%xmm8 + paddd 64(%rsp),%xmm8 // accumulate key material paddd 80(%rsp),%xmm9 paddd 96(%rsp),%xmm10 paddd 112(%rsp),%xmm11 - movdqa %xmm8,%xmm6 + movdqa %xmm8,%xmm6 // "de-interlace" data punpckldq %xmm9,%xmm8 movdqa %xmm10,%xmm7 punpckldq %xmm11,%xmm10 punpckhdq %xmm9,%xmm6 punpckhdq %xmm11,%xmm7 movdqa %xmm8,%xmm9 - punpcklqdq %xmm10,%xmm8 + punpcklqdq %xmm10,%xmm8 // "a0" movdqa %xmm6,%xmm11 - punpcklqdq %xmm7,%xmm6 - punpckhqdq %xmm10,%xmm9 - punpckhqdq %xmm7,%xmm11 + punpcklqdq %xmm7,%xmm6 // "a2" + punpckhqdq %xmm10,%xmm9 // "a1" + punpckhqdq %xmm7,%xmm11 // "a3" paddd 128-256(%rcx),%xmm12 paddd 144-256(%rcx),%xmm13 paddd 160-256(%rcx),%xmm14 paddd 176-256(%rcx),%xmm15 - movdqa %xmm8,0(%rsp) + movdqa %xmm8,0(%rsp) // offload movdqa %xmm9,16(%rsp) - movdqa 32(%rsp),%xmm8 - movdqa 48(%rsp),%xmm9 + movdqa 32(%rsp),%xmm8 // "xc2" + movdqa 48(%rsp),%xmm9 // "xc3" movdqa %xmm12,%xmm10 punpckldq %xmm13,%xmm12 @@ -741,17 +741,17 @@ punpckhdq %xmm13,%xmm10 punpckhdq %xmm15,%xmm7 movdqa %xmm12,%xmm13 - punpcklqdq %xmm14,%xmm12 + punpcklqdq %xmm14,%xmm12 // "b0" movdqa %xmm10,%xmm15 - punpcklqdq %xmm7,%xmm10 - punpckhqdq %xmm14,%xmm13 - punpckhqdq %xmm7,%xmm15 + punpcklqdq %xmm7,%xmm10 // "b2" + punpckhqdq %xmm14,%xmm13 // "b1" + punpckhqdq %xmm7,%xmm15 // "b3" paddd 192-256(%rcx),%xmm4 paddd 208-256(%rcx),%xmm5 paddd 224-256(%rcx),%xmm8 paddd 240-256(%rcx),%xmm9 - movdqa %xmm6,32(%rsp) + movdqa %xmm6,32(%rsp) // keep offloading movdqa %xmm11,48(%rsp) movdqa %xmm4,%xmm14 @@ -761,11 +761,11 @@ punpckhdq %xmm5,%xmm14 punpckhdq %xmm9,%xmm7 movdqa %xmm4,%xmm5 - punpcklqdq %xmm8,%xmm4 + punpcklqdq %xmm8,%xmm4 // "c0" movdqa %xmm14,%xmm9 - punpcklqdq %xmm7,%xmm14 - punpckhqdq %xmm8,%xmm5 - punpckhqdq %xmm7,%xmm9 + punpcklqdq %xmm7,%xmm14 // "c2" + punpckhqdq %xmm8,%xmm5 // "c1" + punpckhqdq %xmm7,%xmm9 // "c3" paddd 256-256(%rcx),%xmm0 paddd 272-256(%rcx),%xmm1 paddd 288-256(%rcx),%xmm2 @@ -778,19 +778,19 @@ punpckhdq %xmm1,%xmm8 punpckhdq %xmm3,%xmm7 movdqa %xmm0,%xmm1 - punpcklqdq %xmm2,%xmm0 + punpcklqdq %xmm2,%xmm0 // "d0" movdqa %xmm8,%xmm3 - punpcklqdq %xmm7,%xmm8 - punpckhqdq %xmm2,%xmm1 - punpckhqdq %xmm7,%xmm3 + punpcklqdq %xmm7,%xmm8 // "d2" + punpckhqdq %xmm2,%xmm1 // "d1" + punpckhqdq %xmm7,%xmm3 // "d3" cmpq $256,%rdx jb .Ltail4x - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -803,7 +803,7 @@ movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 @@ -816,7 +816,7 @@ movdqu %xmm2,96(%rdi) movdqu 32(%rsi),%xmm2 movdqu %xmm7,112(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization movdqu 48(%rsi),%xmm7 pxor 32(%rsp),%xmm6 pxor %xmm10,%xmm11 @@ -831,7 +831,7 @@ movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // inp+=64*4 pxor 48(%rsp),%xmm6 pxor %xmm15,%xmm11 pxor %xmm9,%xmm2 @@ -840,7 +840,7 @@ movdqu %xmm11,80(%rdi) movdqu %xmm2,96(%rdi) movdqu %xmm7,112(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // out+=64*4 subq $256,%rdx jnz .Loop_outer4x @@ -855,9 +855,9 @@ cmpq $64,%rdx jae .L64_or_more4x - +// movdqa 0x00(%rsp),%xmm6 # is offloaded, remember? xorq %r10,%r10 - +// movdqa %xmm6,0x00(%rsp) movdqa %xmm12,16(%rsp) movdqa %xmm4,32(%rsp) movdqa %xmm0,48(%rsp) @@ -865,11 +865,11 @@ .align 32 .L64_or_more4x: - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -879,24 +879,24 @@ movdqu %xmm7,48(%rdi) je .Ldone4x - movdqa 16(%rsp),%xmm6 - leaq 64(%rsi),%rsi + movdqa 16(%rsp),%xmm6 // is offloaded, remember? + leaq 64(%rsi),%rsi // inp+=64*1 xorq %r10,%r10 movdqa %xmm6,0(%rsp) movdqa %xmm13,16(%rsp) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64*1 movdqa %xmm5,32(%rsp) - subq $64,%rdx + subq $64,%rdx // len-=64*1 movdqa %xmm1,48(%rsp) jmp .Loop_tail4x .align 32 .L128_or_more4x: - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -919,24 +919,24 @@ movdqu %xmm7,112(%rdi) je .Ldone4x - movdqa 32(%rsp),%xmm6 - leaq 128(%rsi),%rsi + movdqa 32(%rsp),%xmm6 // is offloaded, remember? + leaq 128(%rsi),%rsi // inp+=64*2 xorq %r10,%r10 movdqa %xmm6,0(%rsp) movdqa %xmm10,16(%rsp) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // out+=64*2 movdqa %xmm14,32(%rsp) - subq $128,%rdx + subq $128,%rdx // len-=64*2 movdqa %xmm8,48(%rsp) jmp .Loop_tail4x .align 32 .L192_or_more4x: - movdqu 0(%rsi),%xmm6 + movdqu 0(%rsi),%xmm6 // xor with input movdqu 16(%rsi),%xmm11 movdqu 32(%rsi),%xmm2 movdqu 48(%rsi),%xmm7 - pxor 0(%rsp),%xmm6 + pxor 0(%rsp),%xmm6 // is offloaded, remember? pxor %xmm12,%xmm11 pxor %xmm4,%xmm2 pxor %xmm0,%xmm7 @@ -949,7 +949,7 @@ movdqu 96(%rsi),%xmm2 movdqu %xmm7,48(%rdi) movdqu 112(%rsi),%xmm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization pxor 16(%rsp),%xmm6 pxor %xmm13,%xmm11 pxor %xmm5,%xmm2 @@ -962,7 +962,7 @@ movdqu %xmm2,96(%rdi) movdqu 32(%rsi),%xmm2 movdqu %xmm7,112(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization movdqu 48(%rsi),%xmm7 pxor 32(%rsp),%xmm6 pxor %xmm10,%xmm11 @@ -974,14 +974,14 @@ movdqu %xmm7,48(%rdi) je .Ldone4x - movdqa 48(%rsp),%xmm6 - leaq 64(%rsi),%rsi + movdqa 48(%rsp),%xmm6 // is offloaded, remember? + leaq 64(%rsi),%rsi // inp+=64*3 xorq %r10,%r10 movdqa %xmm6,0(%rsp) movdqa %xmm15,16(%rsp) - leaq 64(%rdi),%rdi + leaq 64(%rdi),%rdi // out+=64*3 movdqa %xmm9,32(%rsp) - subq $192,%rdx + subq $192,%rdx // len-=64*3 movdqa %xmm3,48(%rsp) .Loop_tail4x: @@ -1007,33 +1007,33 @@ ChaCha20_ctr32_avx2: .cfi_startproc _CET_ENDBR - movq %rsp,%r9 + movq %rsp,%r9 // frame register .cfi_def_cfa_register r9 subq $0x280+8,%rsp andq $-32,%rsp vzeroupper +// ############### stack layout +// +0x00 SIMD equivalent of %r12d +// ... +// +0x80 constant copy of key[0-2] smashed by lanes +// ... +// +0x200 SIMD counters (with nonce smashed by lanes) +// ... +// +0x280 - - - - - - - - - vbroadcasti128 .Lsigma(%rip),%ymm11 - vbroadcasti128 (%rcx),%ymm3 - vbroadcasti128 16(%rcx),%ymm15 - vbroadcasti128 (%r8),%ymm7 - leaq 256(%rsp),%rcx - leaq 512(%rsp),%rax + vbroadcasti128 .Lsigma(%rip),%ymm11 // key[0] + vbroadcasti128 (%rcx),%ymm3 // key[1] + vbroadcasti128 16(%rcx),%ymm15 // key[2] + vbroadcasti128 (%r8),%ymm7 // key[3] + leaq 256(%rsp),%rcx // size optimization + leaq 512(%rsp),%rax // size optimization leaq .Lrot16(%rip),%r10 leaq .Lrot24(%rip),%r11 - vpshufd $0x00,%ymm11,%ymm8 + vpshufd $0x00,%ymm11,%ymm8 // smash key by lanes... vpshufd $0x55,%ymm11,%ymm9 - vmovdqa %ymm8,128-256(%rcx) + vmovdqa %ymm8,128-256(%rcx) // ... and offload vpshufd $0xaa,%ymm11,%ymm10 vmovdqa %ymm9,160-256(%rcx) vpshufd $0xff,%ymm11,%ymm11 @@ -1049,18 +1049,18 @@ vmovdqa %ymm2,320-256(%rcx) vmovdqa %ymm3,352-256(%rcx) - vpshufd $0x00,%ymm15,%ymm12 - vpshufd $0x55,%ymm15,%ymm13 + vpshufd $0x00,%ymm15,%ymm12 // "xc0" + vpshufd $0x55,%ymm15,%ymm13 // "xc1" vmovdqa %ymm12,384-512(%rax) - vpshufd $0xaa,%ymm15,%ymm14 + vpshufd $0xaa,%ymm15,%ymm14 // "xc2" vmovdqa %ymm13,416-512(%rax) - vpshufd $0xff,%ymm15,%ymm15 + vpshufd $0xff,%ymm15,%ymm15 // "xc3" vmovdqa %ymm14,448-512(%rax) vmovdqa %ymm15,480-512(%rax) vpshufd $0x00,%ymm7,%ymm4 vpshufd $0x55,%ymm7,%ymm5 - vpaddd .Lincy(%rip),%ymm4,%ymm4 + vpaddd .Lincy(%rip),%ymm4,%ymm4 // don't save counters yet vpshufd $0xaa,%ymm7,%ymm6 vmovdqa %ymm5,544-512(%rax) vpshufd $0xff,%ymm7,%ymm7 @@ -1071,7 +1071,7 @@ .align 32 .Loop_outer8x: - vmovdqa 128-256(%rcx),%ymm8 + vmovdqa 128-256(%rcx),%ymm8 // re-load smashed key vmovdqa 160-256(%rcx),%ymm9 vmovdqa 192-256(%rcx),%ymm10 vmovdqa 224-256(%rcx),%ymm11 @@ -1079,21 +1079,21 @@ vmovdqa 288-256(%rcx),%ymm1 vmovdqa 320-256(%rcx),%ymm2 vmovdqa 352-256(%rcx),%ymm3 - vmovdqa 384-512(%rax),%ymm12 - vmovdqa 416-512(%rax),%ymm13 - vmovdqa 448-512(%rax),%ymm14 - vmovdqa 480-512(%rax),%ymm15 + vmovdqa 384-512(%rax),%ymm12 // "xc0" + vmovdqa 416-512(%rax),%ymm13 // "xc1" + vmovdqa 448-512(%rax),%ymm14 // "xc2" + vmovdqa 480-512(%rax),%ymm15 // "xc3" vmovdqa 512-512(%rax),%ymm4 vmovdqa 544-512(%rax),%ymm5 vmovdqa 576-512(%rax),%ymm6 vmovdqa 608-512(%rax),%ymm7 - vpaddd .Leight(%rip),%ymm4,%ymm4 + vpaddd .Leight(%rip),%ymm4,%ymm4 // next SIMD counters .Loop_enter8x: - vmovdqa %ymm14,64(%rsp) - vmovdqa %ymm15,96(%rsp) + vmovdqa %ymm14,64(%rsp) // SIMD equivalent of "%nox" + vmovdqa %ymm15,96(%rsp) // SIMD equivalent of "%nox" vbroadcasti128 (%r10),%ymm15 - vmovdqa %ymm4,512-512(%rax) + vmovdqa %ymm4,512-512(%rax) // save SIMD counters movl $10,%eax jmp .Loop8x @@ -1246,20 +1246,20 @@ decl %eax jnz .Loop8x - leaq 512(%rsp),%rax - vpaddd 128-256(%rcx),%ymm8,%ymm8 + leaq 512(%rsp),%rax // size optimization + vpaddd 128-256(%rcx),%ymm8,%ymm8 // accumulate key vpaddd 160-256(%rcx),%ymm9,%ymm9 vpaddd 192-256(%rcx),%ymm10,%ymm10 vpaddd 224-256(%rcx),%ymm11,%ymm11 - vpunpckldq %ymm9,%ymm8,%ymm14 + vpunpckldq %ymm9,%ymm8,%ymm14 // "de-interlace" data vpunpckldq %ymm11,%ymm10,%ymm15 vpunpckhdq %ymm9,%ymm8,%ymm8 vpunpckhdq %ymm11,%ymm10,%ymm10 - vpunpcklqdq %ymm15,%ymm14,%ymm9 - vpunpckhqdq %ymm15,%ymm14,%ymm14 - vpunpcklqdq %ymm10,%ymm8,%ymm11 - vpunpckhqdq %ymm10,%ymm8,%ymm8 + vpunpcklqdq %ymm15,%ymm14,%ymm9 // "a0" + vpunpckhqdq %ymm15,%ymm14,%ymm14 // "a1" + vpunpcklqdq %ymm10,%ymm8,%ymm11 // "a2" + vpunpckhqdq %ymm10,%ymm8,%ymm8 // "a3" vpaddd 256-256(%rcx),%ymm0,%ymm0 vpaddd 288-256(%rcx),%ymm1,%ymm1 vpaddd 320-256(%rcx),%ymm2,%ymm2 @@ -1269,11 +1269,11 @@ vpunpckldq %ymm3,%ymm2,%ymm15 vpunpckhdq %ymm1,%ymm0,%ymm0 vpunpckhdq %ymm3,%ymm2,%ymm2 - vpunpcklqdq %ymm15,%ymm10,%ymm1 - vpunpckhqdq %ymm15,%ymm10,%ymm10 - vpunpcklqdq %ymm2,%ymm0,%ymm3 - vpunpckhqdq %ymm2,%ymm0,%ymm0 - vperm2i128 $0x20,%ymm1,%ymm9,%ymm15 + vpunpcklqdq %ymm15,%ymm10,%ymm1 // "b0" + vpunpckhqdq %ymm15,%ymm10,%ymm10 // "b1" + vpunpcklqdq %ymm2,%ymm0,%ymm3 // "b2" + vpunpckhqdq %ymm2,%ymm0,%ymm0 // "b3" + vperm2i128 $0x20,%ymm1,%ymm9,%ymm15 // "de-interlace" further vperm2i128 $0x31,%ymm1,%ymm9,%ymm1 vperm2i128 $0x20,%ymm10,%ymm14,%ymm9 vperm2i128 $0x31,%ymm10,%ymm14,%ymm10 @@ -1281,10 +1281,10 @@ vperm2i128 $0x31,%ymm3,%ymm11,%ymm3 vperm2i128 $0x20,%ymm0,%ymm8,%ymm11 vperm2i128 $0x31,%ymm0,%ymm8,%ymm0 - vmovdqa %ymm15,0(%rsp) + vmovdqa %ymm15,0(%rsp) // offload vmovdqa %ymm9,32(%rsp) - vmovdqa 64(%rsp),%ymm15 - vmovdqa 96(%rsp),%ymm9 + vmovdqa 64(%rsp),%ymm15 // %ymm15 + vmovdqa 96(%rsp),%ymm9 // %ymm9 vpaddd 384-512(%rax),%ymm12,%ymm12 vpaddd 416-512(%rax),%ymm13,%ymm13 @@ -1295,10 +1295,10 @@ vpunpckldq %ymm9,%ymm15,%ymm8 vpunpckhdq %ymm13,%ymm12,%ymm12 vpunpckhdq %ymm9,%ymm15,%ymm15 - vpunpcklqdq %ymm8,%ymm2,%ymm13 - vpunpckhqdq %ymm8,%ymm2,%ymm2 - vpunpcklqdq %ymm15,%ymm12,%ymm9 - vpunpckhqdq %ymm15,%ymm12,%ymm12 + vpunpcklqdq %ymm8,%ymm2,%ymm13 // "c0" + vpunpckhqdq %ymm8,%ymm2,%ymm2 // "c1" + vpunpcklqdq %ymm15,%ymm12,%ymm9 // "c2" + vpunpckhqdq %ymm15,%ymm12,%ymm12 // "c3" vpaddd 512-512(%rax),%ymm4,%ymm4 vpaddd 544-512(%rax),%ymm5,%ymm5 vpaddd 576-512(%rax),%ymm6,%ymm6 @@ -1308,11 +1308,11 @@ vpunpckldq %ymm7,%ymm6,%ymm8 vpunpckhdq %ymm5,%ymm4,%ymm4 vpunpckhdq %ymm7,%ymm6,%ymm6 - vpunpcklqdq %ymm8,%ymm15,%ymm5 - vpunpckhqdq %ymm8,%ymm15,%ymm15 - vpunpcklqdq %ymm6,%ymm4,%ymm7 - vpunpckhqdq %ymm6,%ymm4,%ymm4 - vperm2i128 $0x20,%ymm5,%ymm13,%ymm8 + vpunpcklqdq %ymm8,%ymm15,%ymm5 // "d0" + vpunpckhqdq %ymm8,%ymm15,%ymm15 // "d1" + vpunpcklqdq %ymm6,%ymm4,%ymm7 // "d2" + vpunpckhqdq %ymm6,%ymm4,%ymm4 // "d3" + vperm2i128 $0x20,%ymm5,%ymm13,%ymm8 // "de-interlace" further vperm2i128 $0x31,%ymm5,%ymm13,%ymm5 vperm2i128 $0x20,%ymm15,%ymm2,%ymm13 vperm2i128 $0x31,%ymm15,%ymm2,%ymm15 @@ -1320,55 +1320,55 @@ vperm2i128 $0x31,%ymm7,%ymm9,%ymm7 vperm2i128 $0x20,%ymm4,%ymm12,%ymm9 vperm2i128 $0x31,%ymm4,%ymm12,%ymm4 - vmovdqa 0(%rsp),%ymm6 + vmovdqa 0(%rsp),%ymm6 // was offloaded, remember? vmovdqa 32(%rsp),%ymm12 cmpq $512,%rdx jb .Ltail8x - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) vmovdqu %ymm1,64(%rdi) vmovdqu %ymm5,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization vpxor 0(%rsi),%ymm12,%ymm12 vpxor 32(%rsi),%ymm13,%ymm13 vpxor 64(%rsi),%ymm10,%ymm10 vpxor 96(%rsi),%ymm15,%ymm15 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm12,0(%rdi) vmovdqu %ymm13,32(%rdi) vmovdqu %ymm10,64(%rdi) vmovdqu %ymm15,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization vpxor 0(%rsi),%ymm14,%ymm14 vpxor 32(%rsi),%ymm2,%ymm2 vpxor 64(%rsi),%ymm3,%ymm3 vpxor 96(%rsi),%ymm7,%ymm7 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm14,0(%rdi) vmovdqu %ymm2,32(%rdi) vmovdqu %ymm3,64(%rdi) vmovdqu %ymm7,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization vpxor 0(%rsi),%ymm11,%ymm11 vpxor 32(%rsi),%ymm9,%ymm9 vpxor 64(%rsi),%ymm0,%ymm0 vpxor 96(%rsi),%ymm4,%ymm4 - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // size optimization vmovdqu %ymm11,0(%rdi) vmovdqu %ymm9,32(%rdi) vmovdqu %ymm0,64(%rdi) vmovdqu %ymm4,96(%rdi) - leaq 128(%rdi),%rdi + leaq 128(%rdi),%rdi // size optimization subq $512,%rdx jnz .Loop_outer8x @@ -1398,23 +1398,23 @@ .align 32 .L64_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vmovdqu %ymm6,0(%rdi) vmovdqu %ymm8,32(%rdi) je .Ldone8x - leaq 64(%rsi),%rsi + leaq 64(%rsi),%rsi // inp+=64*1 xorq %r10,%r10 vmovdqa %ymm1,0(%rsp) - leaq 64(%rdi),%rdi - subq $64,%rdx + leaq 64(%rdi),%rdi // out+=64*1 + subq $64,%rdx // len-=64*1 vmovdqa %ymm5,32(%rsp) jmp .Loop_tail8x .align 32 .L128_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1424,17 +1424,17 @@ vmovdqu %ymm5,96(%rdi) je .Ldone8x - leaq 128(%rsi),%rsi + leaq 128(%rsi),%rsi // inp+=64*2 xorq %r10,%r10 vmovdqa %ymm12,0(%rsp) - leaq 128(%rdi),%rdi - subq $128,%rdx + leaq 128(%rdi),%rdi // out+=64*2 + subq $128,%rdx // len-=64*2 vmovdqa %ymm13,32(%rsp) jmp .Loop_tail8x .align 32 .L192_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1448,17 +1448,17 @@ vmovdqu %ymm13,160(%rdi) je .Ldone8x - leaq 192(%rsi),%rsi + leaq 192(%rsi),%rsi // inp+=64*3 xorq %r10,%r10 vmovdqa %ymm10,0(%rsp) - leaq 192(%rdi),%rdi - subq $192,%rdx + leaq 192(%rdi),%rdi // out+=64*3 + subq $192,%rdx // len-=64*3 vmovdqa %ymm15,32(%rsp) jmp .Loop_tail8x .align 32 .L256_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1476,17 +1476,17 @@ vmovdqu %ymm15,224(%rdi) je .Ldone8x - leaq 256(%rsi),%rsi + leaq 256(%rsi),%rsi // inp+=64*4 xorq %r10,%r10 vmovdqa %ymm14,0(%rsp) - leaq 256(%rdi),%rdi - subq $256,%rdx + leaq 256(%rdi),%rdi // out+=64*4 + subq $256,%rdx // len-=64*4 vmovdqa %ymm2,32(%rsp) jmp .Loop_tail8x .align 32 .L320_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1508,17 +1508,17 @@ vmovdqu %ymm2,288(%rdi) je .Ldone8x - leaq 320(%rsi),%rsi + leaq 320(%rsi),%rsi // inp+=64*5 xorq %r10,%r10 vmovdqa %ymm3,0(%rsp) - leaq 320(%rdi),%rdi - subq $320,%rdx + leaq 320(%rdi),%rdi // out+=64*5 + subq $320,%rdx // len-=64*5 vmovdqa %ymm7,32(%rsp) jmp .Loop_tail8x .align 32 .L384_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1544,17 +1544,17 @@ vmovdqu %ymm7,352(%rdi) je .Ldone8x - leaq 384(%rsi),%rsi + leaq 384(%rsi),%rsi // inp+=64*6 xorq %r10,%r10 vmovdqa %ymm11,0(%rsp) - leaq 384(%rdi),%rdi - subq $384,%rdx + leaq 384(%rdi),%rdi // out+=64*6 + subq $384,%rdx // len-=64*6 vmovdqa %ymm9,32(%rsp) jmp .Loop_tail8x .align 32 .L448_or_more8x: - vpxor 0(%rsi),%ymm6,%ymm6 + vpxor 0(%rsi),%ymm6,%ymm6 // xor with input vpxor 32(%rsi),%ymm8,%ymm8 vpxor 64(%rsi),%ymm1,%ymm1 vpxor 96(%rsi),%ymm5,%ymm5 @@ -1584,11 +1584,11 @@ vmovdqu %ymm9,416(%rdi) je .Ldone8x - leaq 448(%rsi),%rsi + leaq 448(%rsi),%rsi // inp+=64*7 xorq %r10,%r10 vmovdqa %ymm0,0(%rsp) - leaq 448(%rdi),%rdi - subq $448,%rdx + leaq 448(%rdi),%rdi // out+=64*7 + subq $448,%rdx // len-=64*7 vmovdqa %ymm4,32(%rsp) .Loop_tail8x:
diff --git a/gen/crypto/chacha-x86_64-win.asm b/gen/crypto/chacha-x86_64-win.asm index 1becf71..e37e31f 100644 --- a/gen/crypto/chacha-x86_64-win.asm +++ b/gen/crypto/chacha-x86_64-win.asm
@@ -85,25 +85,25 @@ $L$ctr32_body: - +; movdqa .Lsigma(%rip),%xmm0 movdqu xmm1,XMMWORD[rcx] movdqu xmm2,XMMWORD[16+rcx] movdqu xmm3,XMMWORD[r8] movdqa xmm4,XMMWORD[$L$one] - - movdqa XMMWORD[16+rsp],xmm1 - movdqa XMMWORD[32+rsp],xmm2 - movdqa XMMWORD[48+rsp],xmm3 - mov rbp,rdx +; movdqa %xmm0,4*0(%rsp) # key[0] + movdqa XMMWORD[16+rsp],xmm1 ; key[1] + movdqa XMMWORD[32+rsp],xmm2 ; key[2] + movdqa XMMWORD[48+rsp],xmm3 ; key[3] + mov rbp,rdx ; reassign %rdx jmp NEAR $L$oop_outer ALIGN 32 $L$oop_outer: - mov eax,0x61707865 - mov ebx,0x3320646e - mov ecx,0x79622d32 - mov edx,0x6b206574 + mov eax,0x61707865 ; 'expa' + mov ebx,0x3320646e ; 'nd 3' + mov ecx,0x79622d32 ; '2-by' + mov edx,0x6b206574 ; 'te k' mov r8d,DWORD[16+rsp] mov r9d,DWORD[20+rsp] mov r10d,DWORD[24+rsp] @@ -113,13 +113,13 @@ mov r14d,DWORD[56+rsp] mov r15d,DWORD[60+rsp] - mov QWORD[((64+0))+rsp],rbp + mov QWORD[((64+0))+rsp],rbp ; save len mov ebp,10 - mov QWORD[((64+8))+rsp],rsi - movq rsi,xmm2 - mov QWORD[((64+16))+rsp],rdi + mov QWORD[((64+8))+rsp],rsi ; save inp + movq rsi,xmm2 ; "%nox" + mov QWORD[((64+16))+rsp],rdi ; save out mov rdi,rsi - shr rdi,32 + shr rdi,32 ; "%nox" jmp NEAR $L$oop ALIGN 32 @@ -230,18 +230,18 @@ rol r8d,7 dec ebp jnz NEAR $L$oop - mov DWORD[36+rsp],edi + mov DWORD[36+rsp],edi ; modulo-scheduled mov DWORD[32+rsp],esi - mov rbp,QWORD[64+rsp] + mov rbp,QWORD[64+rsp] ; load len movdqa xmm1,xmm2 - mov rsi,QWORD[((64+8))+rsp] - paddd xmm3,xmm4 - mov rdi,QWORD[((64+16))+rsp] + mov rsi,QWORD[((64+8))+rsp] ; load inp + paddd xmm3,xmm4 ; increment counter + mov rdi,QWORD[((64+16))+rsp] ; load out - add eax,0x61707865 - add ebx,0x3320646e - add ecx,0x79622d32 - add edx,0x6b206574 + add eax,0x61707865 ; 'expa' + add ebx,0x3320646e ; 'nd 3' + add ecx,0x79622d32 ; '2-by' + add edx,0x6b206574 ; 'te k' add r8d,DWORD[16+rsp] add r9d,DWORD[20+rsp] add r10d,DWORD[24+rsp] @@ -255,7 +255,7 @@ cmp rbp,64 jb NEAR $L$tail - xor eax,DWORD[rsi] + xor eax,DWORD[rsi] ; xor with input xor ebx,DWORD[4+rsi] xor ecx,DWORD[8+rsi] xor edx,DWORD[12+rsi] @@ -268,13 +268,13 @@ xor r13d,DWORD[52+rsi] xor r14d,DWORD[56+rsi] xor r15d,DWORD[60+rsi] - lea rsi,[64+rsi] + lea rsi,[64+rsi] ; inp+=64 pxor xmm0,xmm1 movdqa XMMWORD[32+rsp],xmm2 movd DWORD[48+rsp],xmm3 - mov DWORD[rdi],eax + mov DWORD[rdi],eax ; write output mov DWORD[4+rdi],ebx mov DWORD[8+rdi],ecx mov DWORD[12+rdi],edx @@ -287,7 +287,7 @@ mov DWORD[52+rdi],r13d mov DWORD[56+rdi],r14d mov DWORD[60+rdi],r15d - lea rdi,[64+rdi] + lea rdi,[64+rdi] ; out+=64 sub rbp,64 jnz NEAR $L$oop_outer @@ -359,7 +359,7 @@ _CET_ENDBR - mov r9,rsp + mov r9,rsp ; frame pointer sub rsp,64+40 movaps XMMWORD[(-40)+r9],xmm6 @@ -376,7 +376,7 @@ movdqa XMMWORD[16+rsp],xmm1 movdqa XMMWORD[32+rsp],xmm2 movdqa XMMWORD[48+rsp],xmm3 - mov r8,10 + mov r8,10 ; reuse %r8 jmp NEAR $L$oop_ssse3 ALIGN 32 @@ -447,19 +447,19 @@ movdqu xmm4,XMMWORD[rsi] movdqu xmm5,XMMWORD[16+rsi] - pxor xmm0,xmm4 + pxor xmm0,xmm4 ; xor with input movdqu xmm4,XMMWORD[32+rsi] pxor xmm1,xmm5 movdqu xmm5,XMMWORD[48+rsi] - lea rsi,[64+rsi] + lea rsi,[64+rsi] ; inp+=64 pxor xmm2,xmm4 pxor xmm3,xmm5 - movdqu XMMWORD[rdi],xmm0 + movdqu XMMWORD[rdi],xmm0 ; write output movdqu XMMWORD[16+rdi],xmm1 movdqu XMMWORD[32+rdi],xmm2 movdqu XMMWORD[48+rdi],xmm3 - lea rdi,[64+rdi] + lea rdi,[64+rdi] ; out+=64 sub rdx,64 jnz NEAR $L$oop_outer_ssse3 @@ -511,7 +511,7 @@ _CET_ENDBR - mov r9,rsp + mov r9,rsp ; frame pointer sub rsp,0x140+168 movaps XMMWORD[(-168)+r9],xmm6 @@ -525,17 +525,17 @@ movaps XMMWORD[(-40)+r9],xmm14 movaps XMMWORD[(-24)+r9],xmm15 $L$4x_body: - movdqa xmm11,XMMWORD[$L$sigma] - movdqu xmm15,XMMWORD[rcx] - movdqu xmm7,XMMWORD[16+rcx] - movdqu xmm3,XMMWORD[r8] - lea rcx,[256+rsp] + movdqa xmm11,XMMWORD[$L$sigma] ; key[0] + movdqu xmm15,XMMWORD[rcx] ; key[1] + movdqu xmm7,XMMWORD[16+rcx] ; key[2] + movdqu xmm3,XMMWORD[r8] ; key[3] + lea rcx,[256+rsp] ; size optimization lea r10,[$L$rot16] lea r11,[$L$rot24] - pshufd xmm8,xmm11,0x00 + pshufd xmm8,xmm11,0x00 ; smash key by lanes... pshufd xmm9,xmm11,0x55 - movdqa XMMWORD[64+rsp],xmm8 + movdqa XMMWORD[64+rsp],xmm8 ; ... and offload pshufd xmm10,xmm11,0xaa movdqa XMMWORD[80+rsp],xmm9 pshufd xmm11,xmm11,0xff @@ -551,18 +551,18 @@ movdqa XMMWORD[(160-256)+rcx],xmm14 movdqa XMMWORD[(176-256)+rcx],xmm15 - pshufd xmm4,xmm7,0x00 - pshufd xmm5,xmm7,0x55 + pshufd xmm4,xmm7,0x00 ; "" + pshufd xmm5,xmm7,0x55 ; "" movdqa XMMWORD[(192-256)+rcx],xmm4 - pshufd xmm6,xmm7,0xaa + pshufd xmm6,xmm7,0xaa ; "" movdqa XMMWORD[(208-256)+rcx],xmm5 - pshufd xmm7,xmm7,0xff + pshufd xmm7,xmm7,0xff ; "" movdqa XMMWORD[(224-256)+rcx],xmm6 movdqa XMMWORD[(240-256)+rcx],xmm7 pshufd xmm0,xmm3,0x00 pshufd xmm1,xmm3,0x55 - paddd xmm0,XMMWORD[$L$inc] + paddd xmm0,XMMWORD[$L$inc] ; don't save counters yet pshufd xmm2,xmm3,0xaa movdqa XMMWORD[(272-256)+rcx],xmm1 pshufd xmm3,xmm3,0xff @@ -573,7 +573,7 @@ ALIGN 32 $L$oop_outer4x: - movdqa xmm8,XMMWORD[64+rsp] + movdqa xmm8,XMMWORD[64+rsp] ; re-load smashed key movdqa xmm9,XMMWORD[80+rsp] movdqa xmm10,XMMWORD[96+rsp] movdqa xmm11,XMMWORD[112+rsp] @@ -581,22 +581,22 @@ movdqa xmm13,XMMWORD[((144-256))+rcx] movdqa xmm14,XMMWORD[((160-256))+rcx] movdqa xmm15,XMMWORD[((176-256))+rcx] - movdqa xmm4,XMMWORD[((192-256))+rcx] - movdqa xmm5,XMMWORD[((208-256))+rcx] - movdqa xmm6,XMMWORD[((224-256))+rcx] - movdqa xmm7,XMMWORD[((240-256))+rcx] + movdqa xmm4,XMMWORD[((192-256))+rcx] ; "" + movdqa xmm5,XMMWORD[((208-256))+rcx] ; "" + movdqa xmm6,XMMWORD[((224-256))+rcx] ; "" + movdqa xmm7,XMMWORD[((240-256))+rcx] ; "" movdqa xmm0,XMMWORD[((256-256))+rcx] movdqa xmm1,XMMWORD[((272-256))+rcx] movdqa xmm2,XMMWORD[((288-256))+rcx] movdqa xmm3,XMMWORD[((304-256))+rcx] - paddd xmm0,XMMWORD[$L$four] + paddd xmm0,XMMWORD[$L$four] ; next SIMD counters $L$oop_enter4x: - movdqa XMMWORD[32+rsp],xmm6 - movdqa XMMWORD[48+rsp],xmm7 - movdqa xmm7,XMMWORD[r10] + movdqa XMMWORD[32+rsp],xmm6 ; SIMD equivalent of "%nox" + movdqa XMMWORD[48+rsp],xmm7 ; SIMD equivalent of "%nox" + movdqa xmm7,XMMWORD[r10] ; .Lrot16(%rip) mov eax,10 - movdqa XMMWORD[(256-256)+rcx],xmm0 + movdqa XMMWORD[(256-256)+rcx],xmm0 ; save SIMD counters jmp NEAR $L$oop4x ALIGN 32 @@ -764,32 +764,32 @@ dec eax jnz NEAR $L$oop4x - paddd xmm8,XMMWORD[64+rsp] + paddd xmm8,XMMWORD[64+rsp] ; accumulate key material paddd xmm9,XMMWORD[80+rsp] paddd xmm10,XMMWORD[96+rsp] paddd xmm11,XMMWORD[112+rsp] - movdqa xmm6,xmm8 + movdqa xmm6,xmm8 ; "de-interlace" data punpckldq xmm8,xmm9 movdqa xmm7,xmm10 punpckldq xmm10,xmm11 punpckhdq xmm6,xmm9 punpckhdq xmm7,xmm11 movdqa xmm9,xmm8 - punpcklqdq xmm8,xmm10 + punpcklqdq xmm8,xmm10 ; "a0" movdqa xmm11,xmm6 - punpcklqdq xmm6,xmm7 - punpckhqdq xmm9,xmm10 - punpckhqdq xmm11,xmm7 + punpcklqdq xmm6,xmm7 ; "a2" + punpckhqdq xmm9,xmm10 ; "a1" + punpckhqdq xmm11,xmm7 ; "a3" paddd xmm12,XMMWORD[((128-256))+rcx] paddd xmm13,XMMWORD[((144-256))+rcx] paddd xmm14,XMMWORD[((160-256))+rcx] paddd xmm15,XMMWORD[((176-256))+rcx] - movdqa XMMWORD[rsp],xmm8 + movdqa XMMWORD[rsp],xmm8 ; offload movdqa XMMWORD[16+rsp],xmm9 - movdqa xmm8,XMMWORD[32+rsp] - movdqa xmm9,XMMWORD[48+rsp] + movdqa xmm8,XMMWORD[32+rsp] ; "xc2" + movdqa xmm9,XMMWORD[48+rsp] ; "xc3" movdqa xmm10,xmm12 punpckldq xmm12,xmm13 @@ -798,17 +798,17 @@ punpckhdq xmm10,xmm13 punpckhdq xmm7,xmm15 movdqa xmm13,xmm12 - punpcklqdq xmm12,xmm14 + punpcklqdq xmm12,xmm14 ; "b0" movdqa xmm15,xmm10 - punpcklqdq xmm10,xmm7 - punpckhqdq xmm13,xmm14 - punpckhqdq xmm15,xmm7 + punpcklqdq xmm10,xmm7 ; "b2" + punpckhqdq xmm13,xmm14 ; "b1" + punpckhqdq xmm15,xmm7 ; "b3" paddd xmm4,XMMWORD[((192-256))+rcx] paddd xmm5,XMMWORD[((208-256))+rcx] paddd xmm8,XMMWORD[((224-256))+rcx] paddd xmm9,XMMWORD[((240-256))+rcx] - movdqa XMMWORD[32+rsp],xmm6 + movdqa XMMWORD[32+rsp],xmm6 ; keep offloading movdqa XMMWORD[48+rsp],xmm11 movdqa xmm14,xmm4 @@ -818,11 +818,11 @@ punpckhdq xmm14,xmm5 punpckhdq xmm7,xmm9 movdqa xmm5,xmm4 - punpcklqdq xmm4,xmm8 + punpcklqdq xmm4,xmm8 ; "c0" movdqa xmm9,xmm14 - punpcklqdq xmm14,xmm7 - punpckhqdq xmm5,xmm8 - punpckhqdq xmm9,xmm7 + punpcklqdq xmm14,xmm7 ; "c2" + punpckhqdq xmm5,xmm8 ; "c1" + punpckhqdq xmm9,xmm7 ; "c3" paddd xmm0,XMMWORD[((256-256))+rcx] paddd xmm1,XMMWORD[((272-256))+rcx] paddd xmm2,XMMWORD[((288-256))+rcx] @@ -835,19 +835,19 @@ punpckhdq xmm8,xmm1 punpckhdq xmm7,xmm3 movdqa xmm1,xmm0 - punpcklqdq xmm0,xmm2 + punpcklqdq xmm0,xmm2 ; "d0" movdqa xmm3,xmm8 - punpcklqdq xmm8,xmm7 - punpckhqdq xmm1,xmm2 - punpckhqdq xmm3,xmm7 + punpcklqdq xmm8,xmm7 ; "d2" + punpckhqdq xmm1,xmm2 ; "d1" + punpckhqdq xmm3,xmm7 ; "d3" cmp rdx,64*4 jb NEAR $L$tail4x - movdqu xmm6,XMMWORD[rsi] + movdqu xmm6,XMMWORD[rsi] ; xor with input movdqu xmm11,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm7,XMMWORD[48+rsi] - pxor xmm6,XMMWORD[rsp] + pxor xmm6,XMMWORD[rsp] ; is offloaded, remember? pxor xmm11,xmm12 pxor xmm2,xmm4 pxor xmm7,xmm0 @@ -860,7 +860,7 @@ movdqu xmm2,XMMWORD[96+rsi] movdqu XMMWORD[48+rdi],xmm7 movdqu xmm7,XMMWORD[112+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; size optimization pxor xmm6,XMMWORD[16+rsp] pxor xmm11,xmm13 pxor xmm2,xmm5 @@ -873,7 +873,7 @@ movdqu XMMWORD[96+rdi],xmm2 movdqu xmm2,XMMWORD[32+rsi] movdqu XMMWORD[112+rdi],xmm7 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; size optimization movdqu xmm7,XMMWORD[48+rsi] pxor xmm6,XMMWORD[32+rsp] pxor xmm11,xmm10 @@ -888,7 +888,7 @@ movdqu xmm2,XMMWORD[96+rsi] movdqu XMMWORD[48+rdi],xmm7 movdqu xmm7,XMMWORD[112+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; inp+=64*4 pxor xmm6,XMMWORD[48+rsp] pxor xmm11,xmm15 pxor xmm2,xmm9 @@ -897,7 +897,7 @@ movdqu XMMWORD[80+rdi],xmm11 movdqu XMMWORD[96+rdi],xmm2 movdqu XMMWORD[112+rdi],xmm7 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; out+=64*4 sub rdx,64*4 jnz NEAR $L$oop_outer4x @@ -912,9 +912,9 @@ cmp rdx,64 jae NEAR $L$64_or_more4x - +; movdqa 0x00(%rsp),%xmm6 # is offloaded, remember? xor r10,r10 - +; movdqa %xmm6,0x00(%rsp) movdqa XMMWORD[16+rsp],xmm12 movdqa XMMWORD[32+rsp],xmm4 movdqa XMMWORD[48+rsp],xmm0 @@ -922,11 +922,11 @@ ALIGN 32 $L$64_or_more4x: - movdqu xmm6,XMMWORD[rsi] + movdqu xmm6,XMMWORD[rsi] ; xor with input movdqu xmm11,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm7,XMMWORD[48+rsi] - pxor xmm6,XMMWORD[rsp] + pxor xmm6,XMMWORD[rsp] ; is offloaded, remember? pxor xmm11,xmm12 pxor xmm2,xmm4 pxor xmm7,xmm0 @@ -936,24 +936,24 @@ movdqu XMMWORD[48+rdi],xmm7 je NEAR $L$done4x - movdqa xmm6,XMMWORD[16+rsp] - lea rsi,[64+rsi] + movdqa xmm6,XMMWORD[16+rsp] ; is offloaded, remember? + lea rsi,[64+rsi] ; inp+=64*1 xor r10,r10 movdqa XMMWORD[rsp],xmm6 movdqa XMMWORD[16+rsp],xmm13 - lea rdi,[64+rdi] + lea rdi,[64+rdi] ; out+=64*1 movdqa XMMWORD[32+rsp],xmm5 - sub rdx,64 + sub rdx,64 ; len-=64*1 movdqa XMMWORD[48+rsp],xmm1 jmp NEAR $L$oop_tail4x ALIGN 32 $L$128_or_more4x: - movdqu xmm6,XMMWORD[rsi] + movdqu xmm6,XMMWORD[rsi] ; xor with input movdqu xmm11,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm7,XMMWORD[48+rsi] - pxor xmm6,XMMWORD[rsp] + pxor xmm6,XMMWORD[rsp] ; is offloaded, remember? pxor xmm11,xmm12 pxor xmm2,xmm4 pxor xmm7,xmm0 @@ -976,24 +976,24 @@ movdqu XMMWORD[112+rdi],xmm7 je NEAR $L$done4x - movdqa xmm6,XMMWORD[32+rsp] - lea rsi,[128+rsi] + movdqa xmm6,XMMWORD[32+rsp] ; is offloaded, remember? + lea rsi,[128+rsi] ; inp+=64*2 xor r10,r10 movdqa XMMWORD[rsp],xmm6 movdqa XMMWORD[16+rsp],xmm10 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; out+=64*2 movdqa XMMWORD[32+rsp],xmm14 - sub rdx,128 + sub rdx,128 ; len-=64*2 movdqa XMMWORD[48+rsp],xmm8 jmp NEAR $L$oop_tail4x ALIGN 32 $L$192_or_more4x: - movdqu xmm6,XMMWORD[rsi] + movdqu xmm6,XMMWORD[rsi] ; xor with input movdqu xmm11,XMMWORD[16+rsi] movdqu xmm2,XMMWORD[32+rsi] movdqu xmm7,XMMWORD[48+rsi] - pxor xmm6,XMMWORD[rsp] + pxor xmm6,XMMWORD[rsp] ; is offloaded, remember? pxor xmm11,xmm12 pxor xmm2,xmm4 pxor xmm7,xmm0 @@ -1006,7 +1006,7 @@ movdqu xmm2,XMMWORD[96+rsi] movdqu XMMWORD[48+rdi],xmm7 movdqu xmm7,XMMWORD[112+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; size optimization pxor xmm6,XMMWORD[16+rsp] pxor xmm11,xmm13 pxor xmm2,xmm5 @@ -1019,7 +1019,7 @@ movdqu XMMWORD[96+rdi],xmm2 movdqu xmm2,XMMWORD[32+rsi] movdqu XMMWORD[112+rdi],xmm7 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; size optimization movdqu xmm7,XMMWORD[48+rsi] pxor xmm6,XMMWORD[32+rsp] pxor xmm11,xmm10 @@ -1031,14 +1031,14 @@ movdqu XMMWORD[48+rdi],xmm7 je NEAR $L$done4x - movdqa xmm6,XMMWORD[48+rsp] - lea rsi,[64+rsi] + movdqa xmm6,XMMWORD[48+rsp] ; is offloaded, remember? + lea rsi,[64+rsi] ; inp+=64*3 xor r10,r10 movdqa XMMWORD[rsp],xmm6 movdqa XMMWORD[16+rsp],xmm15 - lea rdi,[64+rdi] + lea rdi,[64+rdi] ; out+=64*3 movdqa XMMWORD[32+rsp],xmm9 - sub rdx,192 + sub rdx,192 ; len-=64*3 movdqa XMMWORD[48+rsp],xmm3 $L$oop_tail4x: @@ -1086,7 +1086,7 @@ _CET_ENDBR - mov r9,rsp + mov r9,rsp ; frame register sub rsp,0x280+168 and rsp,-32 @@ -1103,27 +1103,27 @@ $L$8x_body: vzeroupper +; ############### stack layout +; +0x00 SIMD equivalent of %r12d +; ... +; +0x80 constant copy of key[0-2] smashed by lanes +; ... +; +0x200 SIMD counters (with nonce smashed by lanes) +; ... +; +0x280 - - - - - - - - - vbroadcasti128 ymm11,XMMWORD[$L$sigma] - vbroadcasti128 ymm3,XMMWORD[rcx] - vbroadcasti128 ymm15,XMMWORD[16+rcx] - vbroadcasti128 ymm7,XMMWORD[r8] - lea rcx,[256+rsp] - lea rax,[512+rsp] + vbroadcasti128 ymm11,XMMWORD[$L$sigma] ; key[0] + vbroadcasti128 ymm3,XMMWORD[rcx] ; key[1] + vbroadcasti128 ymm15,XMMWORD[16+rcx] ; key[2] + vbroadcasti128 ymm7,XMMWORD[r8] ; key[3] + lea rcx,[256+rsp] ; size optimization + lea rax,[512+rsp] ; size optimization lea r10,[$L$rot16] lea r11,[$L$rot24] - vpshufd ymm8,ymm11,0x00 + vpshufd ymm8,ymm11,0x00 ; smash key by lanes... vpshufd ymm9,ymm11,0x55 - vmovdqa YMMWORD[(128-256)+rcx],ymm8 + vmovdqa YMMWORD[(128-256)+rcx],ymm8 ; ... and offload vpshufd ymm10,ymm11,0xaa vmovdqa YMMWORD[(160-256)+rcx],ymm9 vpshufd ymm11,ymm11,0xff @@ -1139,18 +1139,18 @@ vmovdqa YMMWORD[(320-256)+rcx],ymm2 vmovdqa YMMWORD[(352-256)+rcx],ymm3 - vpshufd ymm12,ymm15,0x00 - vpshufd ymm13,ymm15,0x55 + vpshufd ymm12,ymm15,0x00 ; "xc0" + vpshufd ymm13,ymm15,0x55 ; "xc1" vmovdqa YMMWORD[(384-512)+rax],ymm12 - vpshufd ymm14,ymm15,0xaa + vpshufd ymm14,ymm15,0xaa ; "xc2" vmovdqa YMMWORD[(416-512)+rax],ymm13 - vpshufd ymm15,ymm15,0xff + vpshufd ymm15,ymm15,0xff ; "xc3" vmovdqa YMMWORD[(448-512)+rax],ymm14 vmovdqa YMMWORD[(480-512)+rax],ymm15 vpshufd ymm4,ymm7,0x00 vpshufd ymm5,ymm7,0x55 - vpaddd ymm4,ymm4,YMMWORD[$L$incy] + vpaddd ymm4,ymm4,YMMWORD[$L$incy] ; don't save counters yet vpshufd ymm6,ymm7,0xaa vmovdqa YMMWORD[(544-512)+rax],ymm5 vpshufd ymm7,ymm7,0xff @@ -1161,7 +1161,7 @@ ALIGN 32 $L$oop_outer8x: - vmovdqa ymm8,YMMWORD[((128-256))+rcx] + vmovdqa ymm8,YMMWORD[((128-256))+rcx] ; re-load smashed key vmovdqa ymm9,YMMWORD[((160-256))+rcx] vmovdqa ymm10,YMMWORD[((192-256))+rcx] vmovdqa ymm11,YMMWORD[((224-256))+rcx] @@ -1169,21 +1169,21 @@ vmovdqa ymm1,YMMWORD[((288-256))+rcx] vmovdqa ymm2,YMMWORD[((320-256))+rcx] vmovdqa ymm3,YMMWORD[((352-256))+rcx] - vmovdqa ymm12,YMMWORD[((384-512))+rax] - vmovdqa ymm13,YMMWORD[((416-512))+rax] - vmovdqa ymm14,YMMWORD[((448-512))+rax] - vmovdqa ymm15,YMMWORD[((480-512))+rax] + vmovdqa ymm12,YMMWORD[((384-512))+rax] ; "xc0" + vmovdqa ymm13,YMMWORD[((416-512))+rax] ; "xc1" + vmovdqa ymm14,YMMWORD[((448-512))+rax] ; "xc2" + vmovdqa ymm15,YMMWORD[((480-512))+rax] ; "xc3" vmovdqa ymm4,YMMWORD[((512-512))+rax] vmovdqa ymm5,YMMWORD[((544-512))+rax] vmovdqa ymm6,YMMWORD[((576-512))+rax] vmovdqa ymm7,YMMWORD[((608-512))+rax] - vpaddd ymm4,ymm4,YMMWORD[$L$eight] + vpaddd ymm4,ymm4,YMMWORD[$L$eight] ; next SIMD counters $L$oop_enter8x: - vmovdqa YMMWORD[64+rsp],ymm14 - vmovdqa YMMWORD[96+rsp],ymm15 + vmovdqa YMMWORD[64+rsp],ymm14 ; SIMD equivalent of "%nox" + vmovdqa YMMWORD[96+rsp],ymm15 ; SIMD equivalent of "%nox" vbroadcasti128 ymm15,XMMWORD[r10] - vmovdqa YMMWORD[(512-512)+rax],ymm4 + vmovdqa YMMWORD[(512-512)+rax],ymm4 ; save SIMD counters mov eax,10 jmp NEAR $L$oop8x @@ -1336,20 +1336,20 @@ dec eax jnz NEAR $L$oop8x - lea rax,[512+rsp] - vpaddd ymm8,ymm8,YMMWORD[((128-256))+rcx] + lea rax,[512+rsp] ; size optimization + vpaddd ymm8,ymm8,YMMWORD[((128-256))+rcx] ; accumulate key vpaddd ymm9,ymm9,YMMWORD[((160-256))+rcx] vpaddd ymm10,ymm10,YMMWORD[((192-256))+rcx] vpaddd ymm11,ymm11,YMMWORD[((224-256))+rcx] - vpunpckldq ymm14,ymm8,ymm9 + vpunpckldq ymm14,ymm8,ymm9 ; "de-interlace" data vpunpckldq ymm15,ymm10,ymm11 vpunpckhdq ymm8,ymm8,ymm9 vpunpckhdq ymm10,ymm10,ymm11 - vpunpcklqdq ymm9,ymm14,ymm15 - vpunpckhqdq ymm14,ymm14,ymm15 - vpunpcklqdq ymm11,ymm8,ymm10 - vpunpckhqdq ymm8,ymm8,ymm10 + vpunpcklqdq ymm9,ymm14,ymm15 ; "a0" + vpunpckhqdq ymm14,ymm14,ymm15 ; "a1" + vpunpcklqdq ymm11,ymm8,ymm10 ; "a2" + vpunpckhqdq ymm8,ymm8,ymm10 ; "a3" vpaddd ymm0,ymm0,YMMWORD[((256-256))+rcx] vpaddd ymm1,ymm1,YMMWORD[((288-256))+rcx] vpaddd ymm2,ymm2,YMMWORD[((320-256))+rcx] @@ -1359,11 +1359,11 @@ vpunpckldq ymm15,ymm2,ymm3 vpunpckhdq ymm0,ymm0,ymm1 vpunpckhdq ymm2,ymm2,ymm3 - vpunpcklqdq ymm1,ymm10,ymm15 - vpunpckhqdq ymm10,ymm10,ymm15 - vpunpcklqdq ymm3,ymm0,ymm2 - vpunpckhqdq ymm0,ymm0,ymm2 - vperm2i128 ymm15,ymm9,ymm1,0x20 + vpunpcklqdq ymm1,ymm10,ymm15 ; "b0" + vpunpckhqdq ymm10,ymm10,ymm15 ; "b1" + vpunpcklqdq ymm3,ymm0,ymm2 ; "b2" + vpunpckhqdq ymm0,ymm0,ymm2 ; "b3" + vperm2i128 ymm15,ymm9,ymm1,0x20 ; "de-interlace" further vperm2i128 ymm1,ymm9,ymm1,0x31 vperm2i128 ymm9,ymm14,ymm10,0x20 vperm2i128 ymm10,ymm14,ymm10,0x31 @@ -1371,10 +1371,10 @@ vperm2i128 ymm3,ymm11,ymm3,0x31 vperm2i128 ymm11,ymm8,ymm0,0x20 vperm2i128 ymm0,ymm8,ymm0,0x31 - vmovdqa YMMWORD[rsp],ymm15 + vmovdqa YMMWORD[rsp],ymm15 ; offload vmovdqa YMMWORD[32+rsp],ymm9 - vmovdqa ymm15,YMMWORD[64+rsp] - vmovdqa ymm9,YMMWORD[96+rsp] + vmovdqa ymm15,YMMWORD[64+rsp] ; %ymm15 + vmovdqa ymm9,YMMWORD[96+rsp] ; %ymm9 vpaddd ymm12,ymm12,YMMWORD[((384-512))+rax] vpaddd ymm13,ymm13,YMMWORD[((416-512))+rax] @@ -1385,10 +1385,10 @@ vpunpckldq ymm8,ymm15,ymm9 vpunpckhdq ymm12,ymm12,ymm13 vpunpckhdq ymm15,ymm15,ymm9 - vpunpcklqdq ymm13,ymm2,ymm8 - vpunpckhqdq ymm2,ymm2,ymm8 - vpunpcklqdq ymm9,ymm12,ymm15 - vpunpckhqdq ymm12,ymm12,ymm15 + vpunpcklqdq ymm13,ymm2,ymm8 ; "c0" + vpunpckhqdq ymm2,ymm2,ymm8 ; "c1" + vpunpcklqdq ymm9,ymm12,ymm15 ; "c2" + vpunpckhqdq ymm12,ymm12,ymm15 ; "c3" vpaddd ymm4,ymm4,YMMWORD[((512-512))+rax] vpaddd ymm5,ymm5,YMMWORD[((544-512))+rax] vpaddd ymm6,ymm6,YMMWORD[((576-512))+rax] @@ -1398,11 +1398,11 @@ vpunpckldq ymm8,ymm6,ymm7 vpunpckhdq ymm4,ymm4,ymm5 vpunpckhdq ymm6,ymm6,ymm7 - vpunpcklqdq ymm5,ymm15,ymm8 - vpunpckhqdq ymm15,ymm15,ymm8 - vpunpcklqdq ymm7,ymm4,ymm6 - vpunpckhqdq ymm4,ymm4,ymm6 - vperm2i128 ymm8,ymm13,ymm5,0x20 + vpunpcklqdq ymm5,ymm15,ymm8 ; "d0" + vpunpckhqdq ymm15,ymm15,ymm8 ; "d1" + vpunpcklqdq ymm7,ymm4,ymm6 ; "d2" + vpunpckhqdq ymm4,ymm4,ymm6 ; "d3" + vperm2i128 ymm8,ymm13,ymm5,0x20 ; "de-interlace" further vperm2i128 ymm5,ymm13,ymm5,0x31 vperm2i128 ymm13,ymm2,ymm15,0x20 vperm2i128 ymm15,ymm2,ymm15,0x31 @@ -1410,55 +1410,55 @@ vperm2i128 ymm7,ymm9,ymm7,0x31 vperm2i128 ymm9,ymm12,ymm4,0x20 vperm2i128 ymm4,ymm12,ymm4,0x31 - vmovdqa ymm6,YMMWORD[rsp] + vmovdqa ymm6,YMMWORD[rsp] ; was offloaded, remember? vmovdqa ymm12,YMMWORD[32+rsp] cmp rdx,64*8 jb NEAR $L$tail8x - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; size optimization vmovdqu YMMWORD[rdi],ymm6 vmovdqu YMMWORD[32+rdi],ymm8 vmovdqu YMMWORD[64+rdi],ymm1 vmovdqu YMMWORD[96+rdi],ymm5 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; size optimization vpxor ymm12,ymm12,YMMWORD[rsi] vpxor ymm13,ymm13,YMMWORD[32+rsi] vpxor ymm10,ymm10,YMMWORD[64+rsi] vpxor ymm15,ymm15,YMMWORD[96+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; size optimization vmovdqu YMMWORD[rdi],ymm12 vmovdqu YMMWORD[32+rdi],ymm13 vmovdqu YMMWORD[64+rdi],ymm10 vmovdqu YMMWORD[96+rdi],ymm15 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; size optimization vpxor ymm14,ymm14,YMMWORD[rsi] vpxor ymm2,ymm2,YMMWORD[32+rsi] vpxor ymm3,ymm3,YMMWORD[64+rsi] vpxor ymm7,ymm7,YMMWORD[96+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; size optimization vmovdqu YMMWORD[rdi],ymm14 vmovdqu YMMWORD[32+rdi],ymm2 vmovdqu YMMWORD[64+rdi],ymm3 vmovdqu YMMWORD[96+rdi],ymm7 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; size optimization vpxor ymm11,ymm11,YMMWORD[rsi] vpxor ymm9,ymm9,YMMWORD[32+rsi] vpxor ymm0,ymm0,YMMWORD[64+rsi] vpxor ymm4,ymm4,YMMWORD[96+rsi] - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; size optimization vmovdqu YMMWORD[rdi],ymm11 vmovdqu YMMWORD[32+rdi],ymm9 vmovdqu YMMWORD[64+rdi],ymm0 vmovdqu YMMWORD[96+rdi],ymm4 - lea rdi,[128+rdi] + lea rdi,[128+rdi] ; size optimization sub rdx,64*8 jnz NEAR $L$oop_outer8x @@ -1488,23 +1488,23 @@ ALIGN 32 $L$64_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vmovdqu YMMWORD[rdi],ymm6 vmovdqu YMMWORD[32+rdi],ymm8 je NEAR $L$done8x - lea rsi,[64+rsi] + lea rsi,[64+rsi] ; inp+=64*1 xor r10,r10 vmovdqa YMMWORD[rsp],ymm1 - lea rdi,[64+rdi] - sub rdx,64 + lea rdi,[64+rdi] ; out+=64*1 + sub rdx,64 ; len-=64*1 vmovdqa YMMWORD[32+rsp],ymm5 jmp NEAR $L$oop_tail8x ALIGN 32 $L$128_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] @@ -1514,17 +1514,17 @@ vmovdqu YMMWORD[96+rdi],ymm5 je NEAR $L$done8x - lea rsi,[128+rsi] + lea rsi,[128+rsi] ; inp+=64*2 xor r10,r10 vmovdqa YMMWORD[rsp],ymm12 - lea rdi,[128+rdi] - sub rdx,128 + lea rdi,[128+rdi] ; out+=64*2 + sub rdx,128 ; len-=64*2 vmovdqa YMMWORD[32+rsp],ymm13 jmp NEAR $L$oop_tail8x ALIGN 32 $L$192_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] @@ -1538,17 +1538,17 @@ vmovdqu YMMWORD[160+rdi],ymm13 je NEAR $L$done8x - lea rsi,[192+rsi] + lea rsi,[192+rsi] ; inp+=64*3 xor r10,r10 vmovdqa YMMWORD[rsp],ymm10 - lea rdi,[192+rdi] - sub rdx,192 + lea rdi,[192+rdi] ; out+=64*3 + sub rdx,192 ; len-=64*3 vmovdqa YMMWORD[32+rsp],ymm15 jmp NEAR $L$oop_tail8x ALIGN 32 $L$256_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] @@ -1566,17 +1566,17 @@ vmovdqu YMMWORD[224+rdi],ymm15 je NEAR $L$done8x - lea rsi,[256+rsi] + lea rsi,[256+rsi] ; inp+=64*4 xor r10,r10 vmovdqa YMMWORD[rsp],ymm14 - lea rdi,[256+rdi] - sub rdx,256 + lea rdi,[256+rdi] ; out+=64*4 + sub rdx,256 ; len-=64*4 vmovdqa YMMWORD[32+rsp],ymm2 jmp NEAR $L$oop_tail8x ALIGN 32 $L$320_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] @@ -1598,17 +1598,17 @@ vmovdqu YMMWORD[288+rdi],ymm2 je NEAR $L$done8x - lea rsi,[320+rsi] + lea rsi,[320+rsi] ; inp+=64*5 xor r10,r10 vmovdqa YMMWORD[rsp],ymm3 - lea rdi,[320+rdi] - sub rdx,320 + lea rdi,[320+rdi] ; out+=64*5 + sub rdx,320 ; len-=64*5 vmovdqa YMMWORD[32+rsp],ymm7 jmp NEAR $L$oop_tail8x ALIGN 32 $L$384_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] @@ -1634,17 +1634,17 @@ vmovdqu YMMWORD[352+rdi],ymm7 je NEAR $L$done8x - lea rsi,[384+rsi] + lea rsi,[384+rsi] ; inp+=64*6 xor r10,r10 vmovdqa YMMWORD[rsp],ymm11 - lea rdi,[384+rdi] - sub rdx,384 + lea rdi,[384+rdi] ; out+=64*6 + sub rdx,384 ; len-=64*6 vmovdqa YMMWORD[32+rsp],ymm9 jmp NEAR $L$oop_tail8x ALIGN 32 $L$448_or_more8x: - vpxor ymm6,ymm6,YMMWORD[rsi] + vpxor ymm6,ymm6,YMMWORD[rsi] ; xor with input vpxor ymm8,ymm8,YMMWORD[32+rsi] vpxor ymm1,ymm1,YMMWORD[64+rsi] vpxor ymm5,ymm5,YMMWORD[96+rsi] @@ -1674,11 +1674,11 @@ vmovdqu YMMWORD[416+rdi],ymm9 je NEAR $L$done8x - lea rsi,[448+rsi] + lea rsi,[448+rsi] ; inp+=64*7 xor r10,r10 vmovdqa YMMWORD[rsp],ymm0 - lea rdi,[448+rdi] - sub rdx,448 + lea rdi,[448+rdi] ; out+=64*7 + sub rdx,448 ; len-=64*7 vmovdqa YMMWORD[32+rsp],ymm4 $L$oop_tail8x: @@ -1725,20 +1725,20 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData lea r10,[$L$ctr32_body] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<.Lprologue jb NEAR $L$common_seh_tail - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp - lea r10,[$L$no_data] - cmp rbx,r10 + lea r10,[$L$no_data] ; epilogue label + cmp rbx,r10 ; context->Rip>=.Lepilogue jae NEAR $L$common_seh_tail lea rax,[((64+24+48))+rax] @@ -1749,40 +1749,40 @@ mov r13,QWORD[((-32))+rax] mov r14,QWORD[((-40))+rax] mov r15,QWORD[((-48))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[224+r8],r13 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[224+r8],r13 ; restore context->R13 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R14 $L$common_seh_tail: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15 @@ -1810,28 +1810,28 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[192+r8] + mov rax,QWORD[192+r8] ; pull context->R9 - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail lea rsi,[((-40))+rax] - lea rdi,[512+r8] + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,4 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq jmp NEAR $L$common_seh_tail @@ -1850,28 +1850,28 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip - mov rsi,QWORD[8+r9] - mov r11,QWORD[56+r9] + mov rsi,QWORD[8+r9] ; disp->ImageBase + mov r11,QWORD[56+r9] ; disp->HandlerData - mov r10d,DWORD[r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[r11] ; HandlerData[0] + lea r10,[r10*1+rsi] ; prologue label + cmp rbx,r10 ; context->Rip<prologue label jb NEAR $L$common_seh_tail - mov rax,QWORD[192+r8] + mov rax,QWORD[192+r8] ; pull context->R9 - mov r10d,DWORD[4+r11] - lea r10,[r10*1+rsi] - cmp rbx,r10 + mov r10d,DWORD[4+r11] ; HandlerData[1] + lea r10,[r10*1+rsi] ; epilogue label + cmp rbx,r10 ; context->Rip>=epilogue label jae NEAR $L$common_seh_tail lea rsi,[((-168))+rax] - lea rdi,[512+r8] + lea rdi,[512+r8] ; &context.Xmm6 mov ecx,20 - DD 0xa548f3fc + DD 0xa548f3fc ; cld; rep movsq jmp NEAR $L$common_seh_tail @@ -1910,7 +1910,7 @@ $L$SEH_info_ChaCha20_ctr32_avx2: DB 9,0,0,0 DD full_handler wrt ..imagebase - DD $L$8x_body wrt ..imagebase,$L$8x_epilogue wrt ..imagebase + DD $L$8x_body wrt ..imagebase,$L$8x_epilogue wrt ..imagebase ; HandlerData[] %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/crypto/chacha20_poly1305_x86_64-apple.S b/gen/crypto/chacha20_poly1305_x86_64-apple.S index 56f17af..dbfa00e 100644 --- a/gen/crypto/chacha20_poly1305_x86_64-apple.S +++ b/gen/crypto/chacha20_poly1305_x86_64-apple.S
@@ -56,7 +56,7 @@ cmpq $13,%r8 jne L$hash_ad_loop L$poly_fast_tls_ad: - +// Special treatment for the TLS case of 13 bytes movq (%rcx),%r10 movq 5(%rcx),%r11 shrq $24,%r11 @@ -87,21 +87,21 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 ret L$hash_ad_loop: - +// Hash in 16 byte chunk cmpq $16,%r8 jb L$hash_ad_tail addq 0+0(%rcx),%r10 @@ -133,14 +133,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -151,7 +151,7 @@ L$hash_ad_tail: cmpq $0,%r8 je L$hash_ad_done - +// Hash last < 16 byte tail xorq %r13,%r13 xorq %r14,%r14 xorq %r15,%r15 @@ -194,19 +194,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Finished AD L$hash_ad_done: ret @@ -232,8 +232,8 @@ pushq %r15 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 subq $288 + 0 + 32,%rsp @@ -248,14 +248,14 @@ cmpq $128,%rbx jbe L$open_sse_128 - +// For long buffers, prepare the poly key first movdqa L$chacha20_consts(%rip),%xmm0 movdqu 0(%r9),%xmm4 movdqu 16(%r9),%xmm8 movdqu 32(%r9),%xmm12 movdqa %xmm12,%xmm7 - +// Store on stack, to free keyp movdqa %xmm4,0+48(%rbp) movdqa %xmm8,0+64(%rbp) movdqa %xmm12,0+96(%rbp) @@ -306,20 +306,20 @@ decq %r10 jne L$open_sse_init_rounds - +// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded paddd L$chacha20_consts(%rip),%xmm0 paddd 0+48(%rbp),%xmm4 - +// Clamp and store the key pand L$clamp(%rip),%xmm0 movdqa %xmm0,0+0(%rbp) movdqa %xmm4,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal L$open_sse_main_loop: cmpq $256,%rbx jb L$open_sse_tail - +// Load state, increment counter blocks movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 movdqa 0+64(%rbp),%xmm8 @@ -345,8 +345,8 @@ movdqa %xmm14,0+128(%rbp) movdqa %xmm15,0+144(%rbp) - - +// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we +// hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16 movq $4,%rcx movq %rsi,%r8 L$open_sse_main_loop_rounds: @@ -481,14 +481,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -608,14 +608,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -694,7 +694,7 @@ subq $256,%rbx jmp L$open_sse_main_loop L$open_sse_tail: - +// Handle the various tail sizes efficiently testq %rbx,%rbx jz L$open_sse_finalize cmpq $192,%rbx @@ -744,14 +744,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -812,7 +812,7 @@ paddd 0+96(%rbp),%xmm12 jmp L$open_sse_tail_64_dec_loop - +// ############################################################################# L$open_sse_tail_128: movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -860,14 +860,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -988,7 +988,7 @@ leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi jmp L$open_sse_tail_64_dec_loop - +// ############################################################################# L$open_sse_tail_192: movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -1045,14 +1045,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1221,14 +1221,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1264,14 +1264,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1318,7 +1318,7 @@ leaq 128(%rsi),%rsi leaq 128(%rdi),%rdi jmp L$open_sse_tail_64_dec_loop - +// ############################################################################# L$open_sse_tail_256: movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -1529,14 +1529,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1600,14 +1600,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1673,8 +1673,8 @@ subq $192,%rbx leaq 192(%rsi),%rsi leaq 192(%rdi),%rdi - - +// ############################################################################# +// Decrypt the remaining data, 16B at a time, using existing stream L$open_sse_tail_64_dec_loop: cmpq $16,%rbx jb L$open_sse_tail_16_init @@ -1691,13 +1691,13 @@ L$open_sse_tail_16_init: movdqa %xmm0,%xmm1 - +// Decrypt up to 16 bytes at the end. L$open_sse_tail_16: testq %rbx,%rbx jz L$open_sse_finalize - - +// Read the final bytes into %xmm3. They need to be read in reverse order so +// that they end up in the correct order in %xmm3. pxor %xmm3,%xmm3 leaq -1(%rsi,%rbx,1),%rsi movq %rbx,%r8 @@ -1710,10 +1710,10 @@ movq %xmm3,%r13 pextrq $1,%xmm3,%r14 - +// The final bytes of keystream are in %xmm1. pxor %xmm1,%xmm3 - +// Copy the plaintext bytes out. L$open_sse_tail_16_extract: pextrb $0,%xmm3,(%rdi) psrldq $1,%xmm3 @@ -1750,14 +1750,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1793,19 +1793,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Final reduce movq %r10,%r13 movq %r11,%r14 movq %r12,%r15 @@ -1815,14 +1815,14 @@ cmovcq %r13,%r10 cmovcq %r14,%r11 cmovcq %r15,%r12 - +// Add in s part of the key addq 0+0+16(%rbp),%r10 adcq 8+0+16(%rbp),%r11 addq $288 + 0 + 32,%rsp - +// The tag replaces the key on return popq %r9 movq %r10,(%r9) @@ -1840,7 +1840,7 @@ popq %rbp ret - +// ############################################################################# L$open_sse_128: movdqu L$chacha20_consts(%rip),%xmm0 @@ -2003,11 +2003,11 @@ paddd %xmm15,%xmm13 paddd L$sse_inc(%rip),%xmm15 paddd %xmm15,%xmm14 - +// Clamp and store the key pand L$clamp(%rip),%xmm0 movdqa %xmm0,0+0(%rbp) movdqa %xmm4,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal L$open_sse_128_xor_hash: @@ -2018,7 +2018,7 @@ adcq 8+0(%rsi),%r11 adcq $1,%r12 - +// Load for decryption movdqu 0(%rsi),%xmm3 pxor %xmm3,%xmm1 movdqu %xmm1,0(%rdi) @@ -2050,19 +2050,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Shift the stream left movdqa %xmm5,%xmm1 movdqa %xmm9,%xmm5 movdqa %xmm13,%xmm9 @@ -2074,12 +2074,12 @@ - - - - - - +// ############################################################################## +// ############################################################################## +// void chacha20_poly1305_seal(uint8_t *out_ciphertext, const uint8_t *plaintext, +// size_t plaintext_len, const uint8_t *ad, +// size_t ad_len, +// union chacha20_poly1305_seal_data *data); .globl _chacha20_poly1305_seal_sse41 .private_extern _chacha20_poly1305_seal_sse41 @@ -2100,8 +2100,8 @@ pushq %r15 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 subq $288 + 0 + 32,%rsp @@ -2109,7 +2109,7 @@ leaq 32(%rsp),%rbp andq $-32,%rbp - movq 56(%r9),%rbx + movq 56(%r9),%rbx // extra_in_len addq %rdx,%rbx movq %r8,0+0+32(%rbp) movq %rbx,8+0+32(%rbp) @@ -2117,7 +2117,7 @@ cmpq $128,%rbx jbe L$seal_sse_128 - +// For longer buffers, prepare the poly key + some stream movdqa L$chacha20_consts(%rip),%xmm0 movdqu 0(%r9),%xmm4 movdqu 16(%r9),%xmm8 @@ -2138,7 +2138,7 @@ paddd L$sse_inc(%rip),%xmm12 movdqa %xmm12,%xmm13 paddd L$sse_inc(%rip),%xmm12 - +// Store on stack movdqa %xmm4,0+48(%rbp) movdqa %xmm8,0+64(%rbp) movdqa %xmm12,0+96(%rbp) @@ -2351,11 +2351,11 @@ paddd 0+64(%rbp),%xmm8 paddd 0+96(%rbp),%xmm12 - +// Clamp and store the key pand L$clamp(%rip),%xmm3 movdqa %xmm3,0+0(%rbp) movdqa %xmm7,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal movdqu 0 + 0(%rsi),%xmm3 @@ -2572,14 +2572,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2700,14 +2700,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2808,7 +2808,7 @@ ja L$seal_sse_tail_192 cmpq $64,%rbx ja L$seal_sse_tail_128 - +// ############################################################################# L$seal_sse_tail_64: movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -2847,14 +2847,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2932,14 +2932,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2955,7 +2955,7 @@ paddd 0+96(%rbp),%xmm12 jmp L$seal_sse_128_tail_xor - +// ############################################################################# L$seal_sse_tail_128: movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -3000,14 +3000,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3085,14 +3085,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3169,7 +3169,7 @@ subq $64,%rbx leaq 64(%rsi),%rsi jmp L$seal_sse_128_tail_hash - +// ############################################################################# L$seal_sse_tail_192: movdqa L$chacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -3220,14 +3220,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3326,14 +3326,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3446,7 +3446,7 @@ movq $128,%rcx subq $128,%rbx leaq 128(%rsi),%rsi - +// ############################################################################# L$seal_sse_128_tail_hash: cmpq $16,%rcx jb L$seal_sse_128_tail_xor @@ -3479,14 +3479,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3499,11 +3499,11 @@ cmpq $16,%rbx jb L$seal_sse_tail_16 subq $16,%rbx - +// Load for decryption movdqu 0(%rsi),%xmm3 pxor %xmm3,%xmm0 movdqu %xmm0,0(%rdi) - +// Then hash addq 0(%rdi),%r10 adcq 8(%rdi),%r11 adcq $1,%r12 @@ -3535,19 +3535,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Shift the stream left movdqa %xmm4,%xmm0 movdqa %xmm8,%xmm4 movdqa %xmm12,%xmm8 @@ -3560,7 +3560,7 @@ L$seal_sse_tail_16: testq %rbx,%rbx jz L$process_blocks_of_extra_in - +// We can only load the PT one byte at a time to avoid buffer overread movq %rbx,%r8 movq %rbx,%rcx leaq -1(%rsi,%rbx,1),%rsi @@ -3572,10 +3572,10 @@ decq %rcx jne L$seal_sse_tail_16_compose - +// XOR the keystream with the plaintext. pxor %xmm0,%xmm15 - +// Write ciphertext out, byte-by-byte. movq %rbx,%rcx movdqu %xmm15,%xmm0 L$seal_sse_tail_16_extract: @@ -3585,42 +3585,42 @@ subq $1,%rcx jnz L$seal_sse_tail_16_extract - - - - - - - +// %xmm15 contains the final (partial, non-empty) block of ciphertext which +// needs to be fed into the Poly1305 state. The right-most %rbx bytes of it +// are valid. We need to fill it with extra_in bytes until full, or until we +// run out of bytes. +// +// %r9 points to the tag output, which is actually a struct with the +// extra_in pointer and length at offset 48. movq 288 + 0 + 32(%rsp),%r9 - movq 56(%r9),%r14 - movq 48(%r9),%r13 + movq 56(%r9),%r14 // extra_in_len + movq 48(%r9),%r13 // extra_in testq %r14,%r14 - jz L$process_partial_block + jz L$process_partial_block // Common case: no bytes of extra_in movq $16,%r15 - subq %rbx,%r15 - cmpq %r15,%r14 - + subq %rbx,%r15 // 16-%rbx is the number of bytes that fit into %xmm15. + cmpq %r15,%r14 // if extra_in_len < 16-%rbx, only copy extra_in_len +// (note that AT&T syntax reverses the arguments) jge L$load_extra_in movq %r14,%r15 L$load_extra_in: - - +// %r15 contains the number of bytes of extra_in (pointed to by %r13) to load +// into %xmm15. They are loaded in reverse order. leaq -1(%r13,%r15,1),%rsi - - +// Update extra_in and extra_in_len to reflect the bytes that are about to +// be read. addq %r15,%r13 subq %r15,%r14 movq %r13,48(%r9) movq %r14,56(%r9) - - +// Update %r8, which is used to select the mask later on, to reflect the +// extra bytes about to be added. addq %r15,%r8 - +// Load %r15 bytes of extra_in into %xmm11. pxor %xmm11,%xmm11 L$load_extra_load_loop: pslldq $1,%xmm11 @@ -3629,9 +3629,9 @@ subq $1,%r15 jnz L$load_extra_load_loop - - - +// Shift %xmm11 up the length of the remainder from the main encryption. Sadly, +// the shift for an XMM register has to be a constant, thus we loop to do +// this. movq %rbx,%r15 L$load_extra_shift_loop: @@ -3639,18 +3639,18 @@ subq $1,%r15 jnz L$load_extra_shift_loop - - - +// Mask %xmm15 (the remainder from the main encryption) so that superfluous +// bytes are zero. This means that the non-zero bytes in %xmm11 and %xmm15 are +// disjoint and so we can merge them with an OR. leaq L$and_masks(%rip),%r15 shlq $4,%rbx pand -16(%r15,%rbx,1),%xmm15 - +// Merge %xmm11 into %xmm15, forming the remainder block. por %xmm11,%xmm15 - - +// The block of ciphertext + extra_in is ready to be included in the +// Poly1305 state. movq %xmm15,%r13 pextrq $1,%xmm15,%r14 addq %r13,%r10 @@ -3682,26 +3682,26 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 L$process_blocks_of_extra_in: - +// There may be additional bytes of extra_in to process. movq 288+32+0 (%rsp),%r9 - movq 48(%r9),%rsi - movq 56(%r9),%r8 + movq 48(%r9),%rsi // extra_in + movq 56(%r9),%r8 // extra_in_len movq %r8,%rcx - shrq $4,%r8 + shrq $4,%r8 // number of blocks L$process_extra_hash_loop: jz process_extra_in_trailer @@ -3734,14 +3734,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3751,7 +3751,7 @@ jmp L$process_extra_hash_loop .alt_entry process_extra_in_trailer process_extra_in_trailer: - andq $15,%rcx + andq $15,%rcx // remaining num bytes (<16) of extra_in movq %rcx,%rbx jz L$do_length_block leaq -1(%rsi,%rcx,1),%rsi @@ -3764,7 +3764,7 @@ jnz L$process_extra_in_trailer_load L$process_partial_block: - +// %xmm15 contains %rbx bytes of data to be fed into Poly1305. %rbx leaq L$and_masks(%rip),%r15 shlq $4,%rbx pand -16(%r15,%rbx,1),%xmm15 @@ -3799,14 +3799,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3842,19 +3842,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Final reduce movq %r10,%r13 movq %r11,%r14 movq %r12,%r15 @@ -3864,14 +3864,14 @@ cmovcq %r13,%r10 cmovcq %r14,%r11 cmovcq %r15,%r12 - +// Add in s part of the key addq 0+0+16(%rbp),%r10 adcq 8+0+16(%rbp),%r11 addq $288 + 0 + 32,%rsp - +// The tag replaces the key on return popq %r9 movq %r10,(%r9) @@ -3889,7 +3889,7 @@ popq %rbp ret - +// ############################################################################## L$seal_sse_128: movdqu L$chacha20_consts(%rip),%xmm0 @@ -4052,18 +4052,18 @@ paddd %xmm15,%xmm12 paddd L$sse_inc(%rip),%xmm15 paddd %xmm15,%xmm13 - +// Clamp and store the key pand L$clamp(%rip),%xmm2 movdqa %xmm2,0+0(%rbp) movdqa %xmm6,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal jmp L$seal_sse_128_tail_xor - +// ############################################################################# .globl _chacha20_poly1305_open_avx2 .private_extern _chacha20_poly1305_open_avx2 @@ -4084,8 +4084,8 @@ pushq %r15 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 subq $288 + 0 + 32,%rsp @@ -4161,16 +4161,16 @@ vpaddd 0+160(%rbp),%ymm12,%ymm12 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store key vpand L$clamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for the first 64 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 - +// Hash AD + first 64 bytes movq %r8,%r8 call poly_hash_ad_internal - +// Hash first 64 bytes xorq %rcx,%rcx L$open_avx2_init_hash: addq 0+0(%rsi,%rcx,1),%r10 @@ -4202,14 +4202,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4217,17 +4217,17 @@ addq $16,%rcx cmpq $64,%rcx jne L$open_avx2_init_hash - +// Decrypt first 64 bytes vpxor 0(%rsi),%ymm0,%ymm0 vpxor 32(%rsi),%ymm4,%ymm4 - +// Store first 64 bytes of decrypted data vmovdqu %ymm0,0(%rdi) vmovdqu %ymm4,32(%rdi) leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi subq $64,%rbx L$open_avx2_main_loop: - +// Hash and decrypt 512 bytes each iteration cmpq $512,%rbx jb L$open_avx2_main_loop_done vmovdqa L$chacha20_consts(%rip),%ymm0 @@ -4317,14 +4317,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4412,14 +4412,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4507,14 +4507,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4583,14 +4583,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4647,14 +4647,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4697,7 +4697,7 @@ movq %rbx,%rcx andq $-16,%rcx testq %rcx,%rcx - je L$open_avx2_tail_128_rounds + je L$open_avx2_tail_128_rounds // Have nothing to hash L$open_avx2_tail_128_rounds_and_x1hash: addq 0+0(%rsi,%r8,1),%r10 adcq 8+0(%rsi,%r8,1),%r11 @@ -4728,14 +4728,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4796,7 +4796,7 @@ vmovdqa %ymm3,%ymm8 jmp L$open_avx2_tail_128_xor - +// ############################################################################# L$open_avx2_tail_256: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -4842,14 +4842,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4988,14 +4988,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5033,7 +5033,7 @@ leaq 128(%rdi),%rdi subq $128,%rbx jmp L$open_avx2_tail_128_xor - +// ############################################################################# L$open_avx2_tail_384: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -5085,14 +5085,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5185,14 +5185,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5291,14 +5291,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5352,7 +5352,7 @@ leaq 256(%rdi),%rdi subq $256,%rbx jmp L$open_avx2_tail_128_xor - +// ############################################################################# L$open_avx2_tail_512: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -5408,14 +5408,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5481,14 +5481,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5562,14 +5562,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5688,14 +5688,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5786,7 +5786,7 @@ vmovdqa %xmm0,%xmm1 jb L$open_avx2_exit subq $16,%rbx - +// load for decryption vpxor (%rsi),%xmm0,%xmm1 vmovdqu %xmm1,(%rdi) leaq 16(%rsi),%rsi @@ -5796,7 +5796,7 @@ L$open_avx2_exit: vzeroupper jmp L$open_sse_tail_16 - +// ############################################################################# L$open_avx2_192: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -5897,10 +5897,10 @@ vpaddd %ymm11,%ymm12,%ymm12 vpaddd %ymm15,%ymm13,%ymm13 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand L$clamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 192 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -5943,14 +5943,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5983,24 +5983,24 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Load + decrypt vpxor (%rsi),%ymm0,%ymm0 vmovdqu %ymm0,(%rdi) leaq 32(%rsi),%rsi leaq 32(%rdi),%rdi - +// Shift stream vmovdqa %ymm4,%ymm0 vmovdqa %ymm8,%ymm4 vmovdqa %ymm12,%ymm8 @@ -6045,14 +6045,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -6065,7 +6065,7 @@ L$open_avx2_short_tail_32_exit: vzeroupper jmp L$open_sse_tail_16 - +// ############################################################################# L$open_avx2_320: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -6212,10 +6212,10 @@ vpaddd 0+192(%rbp),%ymm13,%ymm13 vpaddd 0+224(%rbp),%ymm14,%ymm14 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand L$clamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 320 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -6229,8 +6229,8 @@ jmp L$open_avx2_short - - +// ############################################################################# +// ############################################################################# .globl _chacha20_poly1305_seal_avx2 .private_extern _chacha20_poly1305_seal_avx2 @@ -6251,8 +6251,8 @@ pushq %r15 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 subq $288 + 0 + 32,%rsp @@ -6260,7 +6260,7 @@ leaq 32(%rsp),%rbp andq $-32,%rbp - movq 56(%r9),%rbx + movq 56(%r9),%rbx // extra_in_len addq %rdx,%rbx movq %r8,0+0+32(%rbp) movq %rbx,8+0+32(%rbp) @@ -6488,7 +6488,7 @@ vmovdqa %ymm15,0+0(%rbp) movq %r8,%r8 call poly_hash_ad_internal - +// Safely store 320 bytes (otherwise would handle with optimized call) vpxor 0(%rsi),%ymm3,%ymm3 vpxor 32(%rsi),%ymm11,%ymm11 vmovdqu %ymm3,0(%rdi) @@ -6868,14 +6868,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -6965,14 +6965,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7060,14 +7060,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7115,14 +7115,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7148,14 +7148,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7239,14 +7239,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7272,14 +7272,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7294,7 +7294,7 @@ ja L$seal_avx2_tail_384 cmpq $128,%rbx ja L$seal_avx2_tail_256 - +// ############################################################################# L$seal_avx2_tail_128: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -7326,14 +7326,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7381,14 +7381,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7433,14 +7433,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7461,7 +7461,7 @@ vmovdqa %ymm3,%ymm8 jmp L$seal_avx2_short_loop - +// ############################################################################# L$seal_avx2_tail_256: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -7505,14 +7505,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7586,14 +7586,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7664,14 +7664,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7711,7 +7711,7 @@ leaq 128(%rsi),%rsi subq $128,%rbx jmp L$seal_avx2_short_hash_remainder - +// ############################################################################# L$seal_avx2_tail_384: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -7760,14 +7760,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7841,14 +7841,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7919,14 +7919,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8020,7 +8020,7 @@ leaq 256(%rsi),%rsi subq $256,%rbx jmp L$seal_avx2_short_hash_remainder - +// ############################################################################# L$seal_avx2_tail_512: vmovdqa L$chacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -8067,14 +8067,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8203,14 +8203,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8336,14 +8336,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8419,7 +8419,7 @@ leaq 384(%rsi),%rsi subq $384,%rbx jmp L$seal_avx2_short_hash_remainder - +// ############################################################################## L$seal_avx2_320: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -8566,10 +8566,10 @@ vpaddd 0+192(%rbp),%ymm13,%ymm13 vpaddd 0+224(%rbp),%ymm14,%ymm14 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand L$clamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 320 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -8581,7 +8581,7 @@ vperm2i128 $0x13,%ymm2,%ymm6,%ymm2 vperm2i128 $0x13,%ymm10,%ymm14,%ymm6 jmp L$seal_avx2_short - +// ############################################################################## L$seal_avx2_192: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -8682,10 +8682,10 @@ vpaddd %ymm11,%ymm12,%ymm12 vpaddd %ymm15,%ymm13,%ymm13 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand L$clamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 192 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -8728,14 +8728,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8747,11 +8747,11 @@ cmpq $32,%rbx jb L$seal_avx2_short_tail subq $32,%rbx - +// Encrypt vpxor (%rsi),%ymm0,%ymm0 vmovdqu %ymm0,(%rdi) leaq 32(%rsi),%rsi - +// Load + hash addq 0+0(%rdi),%r10 adcq 8+0(%rdi),%r11 adcq $1,%r12 @@ -8781,14 +8781,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8821,20 +8821,20 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 leaq 32(%rdi),%rdi - +// Shift stream vmovdqa %ymm4,%ymm0 vmovdqa %ymm8,%ymm4 vmovdqa %ymm12,%ymm8 @@ -8881,14 +8881,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12
diff --git a/gen/crypto/chacha20_poly1305_x86_64-linux.S b/gen/crypto/chacha20_poly1305_x86_64-linux.S index 6fd94c8..9f45a21 100644 --- a/gen/crypto/chacha20_poly1305_x86_64-linux.S +++ b/gen/crypto/chacha20_poly1305_x86_64-linux.S
@@ -56,7 +56,7 @@ cmpq $13,%r8 jne .Lhash_ad_loop .Lpoly_fast_tls_ad: - +// Special treatment for the TLS case of 13 bytes movq (%rcx),%r10 movq 5(%rcx),%r11 shrq $24,%r11 @@ -87,21 +87,21 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 ret .Lhash_ad_loop: - +// Hash in 16 byte chunk cmpq $16,%r8 jb .Lhash_ad_tail addq 0+0(%rcx),%r10 @@ -133,14 +133,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -151,7 +151,7 @@ .Lhash_ad_tail: cmpq $0,%r8 je .Lhash_ad_done - +// Hash last < 16 byte tail xorq %r13,%r13 xorq %r14,%r14 xorq %r15,%r15 @@ -194,19 +194,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Finished AD .Lhash_ad_done: ret .cfi_endproc @@ -237,8 +237,8 @@ pushq %r15 .cfi_adjust_cfa_offset 8 .cfi_offset %r15,-56 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 .cfi_adjust_cfa_offset 8 .cfi_offset %r9,-64 @@ -254,14 +254,14 @@ cmpq $128,%rbx jbe .Lopen_sse_128 - +// For long buffers, prepare the poly key first movdqa .Lchacha20_consts(%rip),%xmm0 movdqu 0(%r9),%xmm4 movdqu 16(%r9),%xmm8 movdqu 32(%r9),%xmm12 movdqa %xmm12,%xmm7 - +// Store on stack, to free keyp movdqa %xmm4,0+48(%rbp) movdqa %xmm8,0+64(%rbp) movdqa %xmm12,0+96(%rbp) @@ -312,20 +312,20 @@ decq %r10 jne .Lopen_sse_init_rounds - +// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded paddd .Lchacha20_consts(%rip),%xmm0 paddd 0+48(%rbp),%xmm4 - +// Clamp and store the key pand .Lclamp(%rip),%xmm0 movdqa %xmm0,0+0(%rbp) movdqa %xmm4,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal .Lopen_sse_main_loop: cmpq $256,%rbx jb .Lopen_sse_tail - +// Load state, increment counter blocks movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 movdqa 0+64(%rbp),%xmm8 @@ -351,8 +351,8 @@ movdqa %xmm14,0+128(%rbp) movdqa %xmm15,0+144(%rbp) - - +// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we +// hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16 movq $4,%rcx movq %rsi,%r8 .Lopen_sse_main_loop_rounds: @@ -487,14 +487,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -614,14 +614,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -700,7 +700,7 @@ subq $256,%rbx jmp .Lopen_sse_main_loop .Lopen_sse_tail: - +// Handle the various tail sizes efficiently testq %rbx,%rbx jz .Lopen_sse_finalize cmpq $192,%rbx @@ -750,14 +750,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -818,7 +818,7 @@ paddd 0+96(%rbp),%xmm12 jmp .Lopen_sse_tail_64_dec_loop - +// ############################################################################# .Lopen_sse_tail_128: movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -866,14 +866,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -994,7 +994,7 @@ leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi jmp .Lopen_sse_tail_64_dec_loop - +// ############################################################################# .Lopen_sse_tail_192: movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -1051,14 +1051,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1227,14 +1227,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1270,14 +1270,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1324,7 +1324,7 @@ leaq 128(%rsi),%rsi leaq 128(%rdi),%rdi jmp .Lopen_sse_tail_64_dec_loop - +// ############################################################################# .Lopen_sse_tail_256: movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -1535,14 +1535,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1606,14 +1606,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1679,8 +1679,8 @@ subq $192,%rbx leaq 192(%rsi),%rsi leaq 192(%rdi),%rdi - - +// ############################################################################# +// Decrypt the remaining data, 16B at a time, using existing stream .Lopen_sse_tail_64_dec_loop: cmpq $16,%rbx jb .Lopen_sse_tail_16_init @@ -1697,13 +1697,13 @@ .Lopen_sse_tail_16_init: movdqa %xmm0,%xmm1 - +// Decrypt up to 16 bytes at the end. .Lopen_sse_tail_16: testq %rbx,%rbx jz .Lopen_sse_finalize - - +// Read the final bytes into %xmm3. They need to be read in reverse order so +// that they end up in the correct order in %xmm3. pxor %xmm3,%xmm3 leaq -1(%rsi,%rbx,1),%rsi movq %rbx,%r8 @@ -1716,10 +1716,10 @@ movq %xmm3,%r13 pextrq $1,%xmm3,%r14 - +// The final bytes of keystream are in %xmm1. pxor %xmm1,%xmm3 - +// Copy the plaintext bytes out. .Lopen_sse_tail_16_extract: pextrb $0,%xmm3,(%rdi) psrldq $1,%xmm3 @@ -1756,14 +1756,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -1799,19 +1799,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Final reduce movq %r10,%r13 movq %r11,%r14 movq %r12,%r15 @@ -1821,14 +1821,14 @@ cmovcq %r13,%r10 cmovcq %r14,%r11 cmovcq %r15,%r12 - +// Add in s part of the key addq 0+0+16(%rbp),%r10 adcq 8+0+16(%rbp),%r11 .cfi_remember_state addq $288 + 0 + 32,%rsp .cfi_adjust_cfa_offset -(288 + 32) - +// The tag replaces the key on return popq %r9 .cfi_adjust_cfa_offset -8 .cfi_restore %r9 @@ -1853,7 +1853,7 @@ .cfi_adjust_cfa_offset -8 .cfi_restore %rbp ret - +// ############################################################################# .Lopen_sse_128: .cfi_restore_state movdqu .Lchacha20_consts(%rip),%xmm0 @@ -2016,11 +2016,11 @@ paddd %xmm15,%xmm13 paddd .Lsse_inc(%rip),%xmm15 paddd %xmm15,%xmm14 - +// Clamp and store the key pand .Lclamp(%rip),%xmm0 movdqa %xmm0,0+0(%rbp) movdqa %xmm4,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal .Lopen_sse_128_xor_hash: @@ -2031,7 +2031,7 @@ adcq 8+0(%rsi),%r11 adcq $1,%r12 - +// Load for decryption movdqu 0(%rsi),%xmm3 pxor %xmm3,%xmm1 movdqu %xmm1,0(%rdi) @@ -2063,19 +2063,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Shift the stream left movdqa %xmm5,%xmm1 movdqa %xmm9,%xmm5 movdqa %xmm13,%xmm9 @@ -2087,12 +2087,12 @@ .size chacha20_poly1305_open_sse41, .-chacha20_poly1305_open_sse41 .cfi_endproc - - - - - - +// ############################################################################## +// ############################################################################## +// void chacha20_poly1305_seal(uint8_t *out_ciphertext, const uint8_t *plaintext, +// size_t plaintext_len, const uint8_t *ad, +// size_t ad_len, +// union chacha20_poly1305_seal_data *data); .globl chacha20_poly1305_seal_sse41 .hidden chacha20_poly1305_seal_sse41 .type chacha20_poly1305_seal_sse41,@function @@ -2118,8 +2118,8 @@ pushq %r15 .cfi_adjust_cfa_offset 8 .cfi_offset %r15,-56 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 .cfi_adjust_cfa_offset 8 .cfi_offset %r9,-64 @@ -2128,7 +2128,7 @@ leaq 32(%rsp),%rbp andq $-32,%rbp - movq 56(%r9),%rbx + movq 56(%r9),%rbx // extra_in_len addq %rdx,%rbx movq %r8,0+0+32(%rbp) movq %rbx,8+0+32(%rbp) @@ -2136,7 +2136,7 @@ cmpq $128,%rbx jbe .Lseal_sse_128 - +// For longer buffers, prepare the poly key + some stream movdqa .Lchacha20_consts(%rip),%xmm0 movdqu 0(%r9),%xmm4 movdqu 16(%r9),%xmm8 @@ -2157,7 +2157,7 @@ paddd .Lsse_inc(%rip),%xmm12 movdqa %xmm12,%xmm13 paddd .Lsse_inc(%rip),%xmm12 - +// Store on stack movdqa %xmm4,0+48(%rbp) movdqa %xmm8,0+64(%rbp) movdqa %xmm12,0+96(%rbp) @@ -2370,11 +2370,11 @@ paddd 0+64(%rbp),%xmm8 paddd 0+96(%rbp),%xmm12 - +// Clamp and store the key pand .Lclamp(%rip),%xmm3 movdqa %xmm3,0+0(%rbp) movdqa %xmm7,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal movdqu 0 + 0(%rsi),%xmm3 @@ -2591,14 +2591,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2719,14 +2719,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2827,7 +2827,7 @@ ja .Lseal_sse_tail_192 cmpq $64,%rbx ja .Lseal_sse_tail_128 - +// ############################################################################# .Lseal_sse_tail_64: movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -2866,14 +2866,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2951,14 +2951,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -2974,7 +2974,7 @@ paddd 0+96(%rbp),%xmm12 jmp .Lseal_sse_128_tail_xor - +// ############################################################################# .Lseal_sse_tail_128: movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -3019,14 +3019,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3104,14 +3104,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3188,7 +3188,7 @@ subq $64,%rbx leaq 64(%rsi),%rsi jmp .Lseal_sse_128_tail_hash - +// ############################################################################# .Lseal_sse_tail_192: movdqa .Lchacha20_consts(%rip),%xmm0 movdqa 0+48(%rbp),%xmm4 @@ -3239,14 +3239,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3345,14 +3345,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3465,7 +3465,7 @@ movq $128,%rcx subq $128,%rbx leaq 128(%rsi),%rsi - +// ############################################################################# .Lseal_sse_128_tail_hash: cmpq $16,%rcx jb .Lseal_sse_128_tail_xor @@ -3498,14 +3498,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3518,11 +3518,11 @@ cmpq $16,%rbx jb .Lseal_sse_tail_16 subq $16,%rbx - +// Load for decryption movdqu 0(%rsi),%xmm3 pxor %xmm3,%xmm0 movdqu %xmm0,0(%rdi) - +// Then hash addq 0(%rdi),%r10 adcq 8(%rdi),%r11 adcq $1,%r12 @@ -3554,19 +3554,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Shift the stream left movdqa %xmm4,%xmm0 movdqa %xmm8,%xmm4 movdqa %xmm12,%xmm8 @@ -3579,7 +3579,7 @@ .Lseal_sse_tail_16: testq %rbx,%rbx jz .Lprocess_blocks_of_extra_in - +// We can only load the PT one byte at a time to avoid buffer overread movq %rbx,%r8 movq %rbx,%rcx leaq -1(%rsi,%rbx,1),%rsi @@ -3591,10 +3591,10 @@ decq %rcx jne .Lseal_sse_tail_16_compose - +// XOR the keystream with the plaintext. pxor %xmm0,%xmm15 - +// Write ciphertext out, byte-by-byte. movq %rbx,%rcx movdqu %xmm15,%xmm0 .Lseal_sse_tail_16_extract: @@ -3604,42 +3604,42 @@ subq $1,%rcx jnz .Lseal_sse_tail_16_extract - - - - - - - +// %xmm15 contains the final (partial, non-empty) block of ciphertext which +// needs to be fed into the Poly1305 state. The right-most %rbx bytes of it +// are valid. We need to fill it with extra_in bytes until full, or until we +// run out of bytes. +// +// %r9 points to the tag output, which is actually a struct with the +// extra_in pointer and length at offset 48. movq 288 + 0 + 32(%rsp),%r9 - movq 56(%r9),%r14 - movq 48(%r9),%r13 + movq 56(%r9),%r14 // extra_in_len + movq 48(%r9),%r13 // extra_in testq %r14,%r14 - jz .Lprocess_partial_block + jz .Lprocess_partial_block // Common case: no bytes of extra_in movq $16,%r15 - subq %rbx,%r15 - cmpq %r15,%r14 - + subq %rbx,%r15 // 16-%rbx is the number of bytes that fit into %xmm15. + cmpq %r15,%r14 // if extra_in_len < 16-%rbx, only copy extra_in_len +// (note that AT&T syntax reverses the arguments) jge .Lload_extra_in movq %r14,%r15 .Lload_extra_in: - - +// %r15 contains the number of bytes of extra_in (pointed to by %r13) to load +// into %xmm15. They are loaded in reverse order. leaq -1(%r13,%r15,1),%rsi - - +// Update extra_in and extra_in_len to reflect the bytes that are about to +// be read. addq %r15,%r13 subq %r15,%r14 movq %r13,48(%r9) movq %r14,56(%r9) - - +// Update %r8, which is used to select the mask later on, to reflect the +// extra bytes about to be added. addq %r15,%r8 - +// Load %r15 bytes of extra_in into %xmm11. pxor %xmm11,%xmm11 .Lload_extra_load_loop: pslldq $1,%xmm11 @@ -3648,9 +3648,9 @@ subq $1,%r15 jnz .Lload_extra_load_loop - - - +// Shift %xmm11 up the length of the remainder from the main encryption. Sadly, +// the shift for an XMM register has to be a constant, thus we loop to do +// this. movq %rbx,%r15 .Lload_extra_shift_loop: @@ -3658,18 +3658,18 @@ subq $1,%r15 jnz .Lload_extra_shift_loop - - - +// Mask %xmm15 (the remainder from the main encryption) so that superfluous +// bytes are zero. This means that the non-zero bytes in %xmm11 and %xmm15 are +// disjoint and so we can merge them with an OR. leaq .Land_masks(%rip),%r15 shlq $4,%rbx pand -16(%r15,%rbx,1),%xmm15 - +// Merge %xmm11 into %xmm15, forming the remainder block. por %xmm11,%xmm15 - - +// The block of ciphertext + extra_in is ready to be included in the +// Poly1305 state. movq %xmm15,%r13 pextrq $1,%xmm15,%r14 addq %r13,%r10 @@ -3701,26 +3701,26 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 .Lprocess_blocks_of_extra_in: - +// There may be additional bytes of extra_in to process. movq 288+32+0 (%rsp),%r9 - movq 48(%r9),%rsi - movq 56(%r9),%r8 + movq 48(%r9),%rsi // extra_in + movq 56(%r9),%r8 // extra_in_len movq %r8,%rcx - shrq $4,%r8 + shrq $4,%r8 // number of blocks .Lprocess_extra_hash_loop: jz process_extra_in_trailer @@ -3753,14 +3753,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3769,7 +3769,7 @@ subq $1,%r8 jmp .Lprocess_extra_hash_loop process_extra_in_trailer: - andq $15,%rcx + andq $15,%rcx // remaining num bytes (<16) of extra_in movq %rcx,%rbx jz .Ldo_length_block leaq -1(%rsi,%rcx,1),%rsi @@ -3782,7 +3782,7 @@ jnz .Lprocess_extra_in_trailer_load .Lprocess_partial_block: - +// %xmm15 contains %rbx bytes of data to be fed into Poly1305. %rbx leaq .Land_masks(%rip),%r15 shlq $4,%rbx pand -16(%r15,%rbx,1),%xmm15 @@ -3817,14 +3817,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -3860,19 +3860,19 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Final reduce movq %r10,%r13 movq %r11,%r14 movq %r12,%r15 @@ -3882,14 +3882,14 @@ cmovcq %r13,%r10 cmovcq %r14,%r11 cmovcq %r15,%r12 - +// Add in s part of the key addq 0+0+16(%rbp),%r10 adcq 8+0+16(%rbp),%r11 .cfi_remember_state addq $288 + 0 + 32,%rsp .cfi_adjust_cfa_offset -(288 + 32) - +// The tag replaces the key on return popq %r9 .cfi_adjust_cfa_offset -8 .cfi_restore %r9 @@ -3914,7 +3914,7 @@ .cfi_adjust_cfa_offset -8 .cfi_restore %rbp ret - +// ############################################################################## .Lseal_sse_128: .cfi_restore_state movdqu .Lchacha20_consts(%rip),%xmm0 @@ -4077,18 +4077,18 @@ paddd %xmm15,%xmm12 paddd .Lsse_inc(%rip),%xmm15 paddd %xmm15,%xmm13 - +// Clamp and store the key pand .Lclamp(%rip),%xmm2 movdqa %xmm2,0+0(%rbp) movdqa %xmm6,0+16(%rbp) - +// Hash movq %r8,%r8 call poly_hash_ad_internal jmp .Lseal_sse_128_tail_xor .size chacha20_poly1305_seal_sse41, .-chacha20_poly1305_seal_sse41 .cfi_endproc - +// ############################################################################# .globl chacha20_poly1305_open_avx2 .hidden chacha20_poly1305_open_avx2 .type chacha20_poly1305_open_avx2,@function @@ -4114,8 +4114,8 @@ pushq %r15 .cfi_adjust_cfa_offset 8 .cfi_offset %r15,-56 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 .cfi_adjust_cfa_offset 8 .cfi_offset %r9,-64 @@ -4192,16 +4192,16 @@ vpaddd 0+160(%rbp),%ymm12,%ymm12 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store key vpand .Lclamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for the first 64 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 - +// Hash AD + first 64 bytes movq %r8,%r8 call poly_hash_ad_internal - +// Hash first 64 bytes xorq %rcx,%rcx .Lopen_avx2_init_hash: addq 0+0(%rsi,%rcx,1),%r10 @@ -4233,14 +4233,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4248,17 +4248,17 @@ addq $16,%rcx cmpq $64,%rcx jne .Lopen_avx2_init_hash - +// Decrypt first 64 bytes vpxor 0(%rsi),%ymm0,%ymm0 vpxor 32(%rsi),%ymm4,%ymm4 - +// Store first 64 bytes of decrypted data vmovdqu %ymm0,0(%rdi) vmovdqu %ymm4,32(%rdi) leaq 64(%rsi),%rsi leaq 64(%rdi),%rdi subq $64,%rbx .Lopen_avx2_main_loop: - +// Hash and decrypt 512 bytes each iteration cmpq $512,%rbx jb .Lopen_avx2_main_loop_done vmovdqa .Lchacha20_consts(%rip),%ymm0 @@ -4348,14 +4348,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4443,14 +4443,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4538,14 +4538,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4614,14 +4614,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4678,14 +4678,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4728,7 +4728,7 @@ movq %rbx,%rcx andq $-16,%rcx testq %rcx,%rcx - je .Lopen_avx2_tail_128_rounds + je .Lopen_avx2_tail_128_rounds // Have nothing to hash .Lopen_avx2_tail_128_rounds_and_x1hash: addq 0+0(%rsi,%r8,1),%r10 adcq 8+0(%rsi,%r8,1),%r11 @@ -4759,14 +4759,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -4827,7 +4827,7 @@ vmovdqa %ymm3,%ymm8 jmp .Lopen_avx2_tail_128_xor - +// ############################################################################# .Lopen_avx2_tail_256: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -4873,14 +4873,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5019,14 +5019,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5064,7 +5064,7 @@ leaq 128(%rdi),%rdi subq $128,%rbx jmp .Lopen_avx2_tail_128_xor - +// ############################################################################# .Lopen_avx2_tail_384: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -5116,14 +5116,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5216,14 +5216,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5322,14 +5322,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5383,7 +5383,7 @@ leaq 256(%rdi),%rdi subq $256,%rbx jmp .Lopen_avx2_tail_128_xor - +// ############################################################################# .Lopen_avx2_tail_512: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -5439,14 +5439,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5512,14 +5512,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5593,14 +5593,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5719,14 +5719,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -5817,7 +5817,7 @@ vmovdqa %xmm0,%xmm1 jb .Lopen_avx2_exit subq $16,%rbx - +// load for decryption vpxor (%rsi),%xmm0,%xmm1 vmovdqu %xmm1,(%rdi) leaq 16(%rsi),%rsi @@ -5827,7 +5827,7 @@ .Lopen_avx2_exit: vzeroupper jmp .Lopen_sse_tail_16 - +// ############################################################################# .Lopen_avx2_192: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -5928,10 +5928,10 @@ vpaddd %ymm11,%ymm12,%ymm12 vpaddd %ymm15,%ymm13,%ymm13 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand .Lclamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 192 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -5974,14 +5974,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -6014,24 +6014,24 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 - +// Load + decrypt vpxor (%rsi),%ymm0,%ymm0 vmovdqu %ymm0,(%rdi) leaq 32(%rsi),%rsi leaq 32(%rdi),%rdi - +// Shift stream vmovdqa %ymm4,%ymm0 vmovdqa %ymm8,%ymm4 vmovdqa %ymm12,%ymm8 @@ -6076,14 +6076,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -6096,7 +6096,7 @@ .Lopen_avx2_short_tail_32_exit: vzeroupper jmp .Lopen_sse_tail_16 - +// ############################################################################# .Lopen_avx2_320: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -6243,10 +6243,10 @@ vpaddd 0+192(%rbp),%ymm13,%ymm13 vpaddd 0+224(%rbp),%ymm14,%ymm14 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand .Lclamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 320 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -6260,8 +6260,8 @@ jmp .Lopen_avx2_short .size chacha20_poly1305_open_avx2, .-chacha20_poly1305_open_avx2 .cfi_endproc - - +// ############################################################################# +// ############################################################################# .globl chacha20_poly1305_seal_avx2 .hidden chacha20_poly1305_seal_avx2 .type chacha20_poly1305_seal_avx2,@function @@ -6287,8 +6287,8 @@ pushq %r15 .cfi_adjust_cfa_offset 8 .cfi_offset %r15,-56 - - +// We write the calculated authenticator back to keyp at the end, so save +// the pointer on the stack too. pushq %r9 .cfi_adjust_cfa_offset 8 .cfi_offset %r9,-64 @@ -6297,7 +6297,7 @@ leaq 32(%rsp),%rbp andq $-32,%rbp - movq 56(%r9),%rbx + movq 56(%r9),%rbx // extra_in_len addq %rdx,%rbx movq %r8,0+0+32(%rbp) movq %rbx,8+0+32(%rbp) @@ -6525,7 +6525,7 @@ vmovdqa %ymm15,0+0(%rbp) movq %r8,%r8 call poly_hash_ad_internal - +// Safely store 320 bytes (otherwise would handle with optimized call) vpxor 0(%rsi),%ymm3,%ymm3 vpxor 32(%rsi),%ymm11,%ymm11 vmovdqu %ymm3,0(%rdi) @@ -6905,14 +6905,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7002,14 +7002,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7097,14 +7097,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7152,14 +7152,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7185,14 +7185,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7276,14 +7276,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7309,14 +7309,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7331,7 +7331,7 @@ ja .Lseal_avx2_tail_384 cmpq $128,%rbx ja .Lseal_avx2_tail_256 - +// ############################################################################# .Lseal_avx2_tail_128: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -7363,14 +7363,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7418,14 +7418,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7470,14 +7470,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7498,7 +7498,7 @@ vmovdqa %ymm3,%ymm8 jmp .Lseal_avx2_short_loop - +// ############################################################################# .Lseal_avx2_tail_256: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -7542,14 +7542,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7623,14 +7623,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7701,14 +7701,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7748,7 +7748,7 @@ leaq 128(%rsi),%rsi subq $128,%rbx jmp .Lseal_avx2_short_hash_remainder - +// ############################################################################# .Lseal_avx2_tail_384: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -7797,14 +7797,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7878,14 +7878,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -7956,14 +7956,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8057,7 +8057,7 @@ leaq 256(%rsi),%rsi subq $256,%rbx jmp .Lseal_avx2_short_hash_remainder - +// ############################################################################# .Lseal_avx2_tail_512: vmovdqa .Lchacha20_consts(%rip),%ymm0 vmovdqa 0+64(%rbp),%ymm4 @@ -8104,14 +8104,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8240,14 +8240,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8373,14 +8373,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8456,7 +8456,7 @@ leaq 384(%rsi),%rsi subq $384,%rbx jmp .Lseal_avx2_short_hash_remainder - +// ############################################################################## .Lseal_avx2_320: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -8603,10 +8603,10 @@ vpaddd 0+192(%rbp),%ymm13,%ymm13 vpaddd 0+224(%rbp),%ymm14,%ymm14 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand .Lclamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 320 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -8618,7 +8618,7 @@ vperm2i128 $0x13,%ymm2,%ymm6,%ymm2 vperm2i128 $0x13,%ymm10,%ymm14,%ymm6 jmp .Lseal_avx2_short - +// ############################################################################## .Lseal_avx2_192: vmovdqa %ymm0,%ymm1 vmovdqa %ymm0,%ymm2 @@ -8719,10 +8719,10 @@ vpaddd %ymm11,%ymm12,%ymm12 vpaddd %ymm15,%ymm13,%ymm13 vperm2i128 $0x02,%ymm0,%ymm4,%ymm3 - +// Clamp and store the key vpand .Lclamp(%rip),%ymm3,%ymm3 vmovdqa %ymm3,0+0(%rbp) - +// Stream for up to 192 bytes vperm2i128 $0x13,%ymm0,%ymm4,%ymm0 vperm2i128 $0x13,%ymm8,%ymm12,%ymm4 vperm2i128 $0x02,%ymm1,%ymm5,%ymm8 @@ -8765,14 +8765,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8784,11 +8784,11 @@ cmpq $32,%rbx jb .Lseal_avx2_short_tail subq $32,%rbx - +// Encrypt vpxor (%rsi),%ymm0,%ymm0 vmovdqu %ymm0,(%rdi) leaq 32(%rsi),%rsi - +// Load + hash addq 0+0(%rdi),%r10 adcq 8+0(%rdi),%r11 adcq $1,%r12 @@ -8818,14 +8818,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 @@ -8858,20 +8858,20 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12 leaq 32(%rdi),%rdi - +// Shift stream vmovdqa %ymm4,%ymm0 vmovdqa %ymm8,%ymm4 vmovdqa %ymm12,%ymm8 @@ -8918,14 +8918,14 @@ movq %r13,%r10 movq %r14,%r11 movq %r15,%r12 - andq $3,%r12 + andq $3,%r12 // At this point acc2 is 2 bits at most (value of 3) movq %r15,%r13 andq $-4,%r13 movq %r9,%r14 shrdq $2,%r9,%r15 shrq $2,%r9 addq %r13,%r15 - adcq %r14,%r9 + adcq %r14,%r9 // No carry out since t3 is 61 bits and t1 is 63 bits addq %r15,%r10 adcq %r9,%r11 adcq $0,%r12
diff --git a/gen/crypto/chacha20_poly1305_x86_64-win.asm b/gen/crypto/chacha20_poly1305_x86_64-win.asm index 4891677..3a3e301 100644 --- a/gen/crypto/chacha20_poly1305_x86_64-win.asm +++ b/gen/crypto/chacha20_poly1305_x86_64-win.asm
@@ -64,7 +64,7 @@ cmp r8,13 jne NEAR $L$hash_ad_loop $L$poly_fast_tls_ad: - +; Special treatment for the TLS case of 13 bytes mov r10,QWORD[rcx] mov r11,QWORD[5+rcx] shr r11,24 @@ -95,21 +95,21 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 ret $L$hash_ad_loop: - +; Hash in 16 byte chunk cmp r8,16 jb NEAR $L$hash_ad_tail add r10,QWORD[((0+0))+rcx] @@ -141,14 +141,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -159,7 +159,7 @@ $L$hash_ad_tail: cmp r8,0 je NEAR $L$hash_ad_done - +; Hash last < 16 byte tail xor r13,r13 xor r14,r14 xor r15,r15 @@ -202,19 +202,19 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 - +; Finished AD $L$hash_ad_done: ret @@ -250,8 +250,8 @@ push r15 - - +; We write the calculated authenticator back to keyp at the end, so save +; the pointer on the stack too. push r9 sub rsp,288 + 160 + 32 @@ -277,14 +277,14 @@ cmp rbx,128 jbe NEAR $L$open_sse_128 - +; For long buffers, prepare the poly key first movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqu xmm4,XMMWORD[r9] movdqu xmm8,XMMWORD[16+r9] movdqu xmm12,XMMWORD[32+r9] movdqa xmm7,xmm12 - +; Store on stack, to free keyp movdqa XMMWORD[(160+48)+rbp],xmm4 movdqa XMMWORD[(160+64)+rbp],xmm8 movdqa XMMWORD[(160+96)+rbp],xmm12 @@ -335,20 +335,20 @@ dec r10 jne NEAR $L$open_sse_init_rounds - +; A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded paddd xmm0,XMMWORD[$L$chacha20_consts] paddd xmm4,XMMWORD[((160+48))+rbp] - +; Clamp and store the key pand xmm0,XMMWORD[$L$clamp] movdqa XMMWORD[(160+0)+rbp],xmm0 movdqa XMMWORD[(160+16)+rbp],xmm4 - +; Hash mov r8,r8 call poly_hash_ad_internal $L$open_sse_main_loop: cmp rbx,16*16 jb NEAR $L$open_sse_tail - +; Load state, increment counter blocks movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] movdqa xmm8,XMMWORD[((160+64))+rbp] @@ -374,8 +374,8 @@ movdqa XMMWORD[(160+128)+rbp],xmm14 movdqa XMMWORD[(160+144)+rbp],xmm15 - - +; There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we +; hash 2 blocks, and for the remaining 4 only 1 block - for a total of 16 mov rcx,4 mov r8,rsi $L$open_sse_main_loop_rounds: @@ -510,14 +510,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -637,14 +637,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -723,7 +723,7 @@ sub rbx,16*16 jmp NEAR $L$open_sse_main_loop $L$open_sse_tail: - +; Handle the various tail sizes efficiently test rbx,rbx jz NEAR $L$open_sse_finalize cmp rbx,12*16 @@ -773,14 +773,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -841,7 +841,7 @@ paddd xmm12,XMMWORD[((160+96))+rbp] jmp NEAR $L$open_sse_tail_64_dec_loop - +; ############################################################################## $L$open_sse_tail_128: movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] @@ -889,14 +889,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1017,7 +1017,7 @@ lea rsi,[64+rsi] lea rdi,[64+rdi] jmp NEAR $L$open_sse_tail_64_dec_loop - +; ############################################################################## $L$open_sse_tail_192: movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] @@ -1074,14 +1074,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1250,14 +1250,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1293,14 +1293,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1347,7 +1347,7 @@ lea rsi,[128+rsi] lea rdi,[128+rdi] jmp NEAR $L$open_sse_tail_64_dec_loop - +; ############################################################################## $L$open_sse_tail_256: movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] @@ -1558,14 +1558,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1629,14 +1629,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1702,8 +1702,8 @@ sub rbx,12*16 lea rsi,[192+rsi] lea rdi,[192+rdi] - - +; ############################################################################## +; Decrypt the remaining data, 16B at a time, using existing stream $L$open_sse_tail_64_dec_loop: cmp rbx,16 jb NEAR $L$open_sse_tail_16_init @@ -1720,13 +1720,13 @@ $L$open_sse_tail_16_init: movdqa xmm1,xmm0 - +; Decrypt up to 16 bytes at the end. $L$open_sse_tail_16: test rbx,rbx jz NEAR $L$open_sse_finalize - - +; Read the final bytes into %xmm3. They need to be read in reverse order so +; that they end up in the correct order in %xmm3. pxor xmm3,xmm3 lea rsi,[((-1))+rbx*1+rsi] mov r8,rbx @@ -1739,10 +1739,10 @@ movq r13,xmm3 pextrq r14,xmm3,1 - +; The final bytes of keystream are in %xmm1. pxor xmm3,xmm1 - +; Copy the plaintext bytes out. $L$open_sse_tail_16_extract: pextrb XMMWORD[rdi],xmm3,0 psrldq xmm3,1 @@ -1779,14 +1779,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -1822,19 +1822,19 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 - +; Final reduce mov r13,r10 mov r14,r11 mov r15,r12 @@ -1844,7 +1844,7 @@ cmovc r10,r13 cmovc r11,r14 cmovc r12,r15 - +; Add in s part of the key add r10,QWORD[((0+160+16))+rbp] adc r11,QWORD[((8+160+16))+rbp] @@ -1862,7 +1862,7 @@ add rsp,288 + 160 + 32 - +; The tag replaces the key on return pop r9 mov QWORD[r9],r10 @@ -1882,7 +1882,7 @@ mov rdi,QWORD[8+rsp] ;WIN64 epilogue mov rsi,QWORD[16+rsp] ret - +; ############################################################################## $L$open_sse_128: movdqu xmm0,XMMWORD[$L$chacha20_consts] @@ -2045,11 +2045,11 @@ paddd xmm13,xmm15 paddd xmm15,XMMWORD[$L$sse_inc] paddd xmm14,xmm15 - +; Clamp and store the key pand xmm0,XMMWORD[$L$clamp] movdqa XMMWORD[(160+0)+rbp],xmm0 movdqa XMMWORD[(160+16)+rbp],xmm4 - +; Hash mov r8,r8 call poly_hash_ad_internal $L$open_sse_128_xor_hash: @@ -2060,7 +2060,7 @@ adc r11,QWORD[((8+0))+rsi] adc r12,1 - +; Load for decryption movdqu xmm3,XMMWORD[rsi] pxor xmm1,xmm3 movdqu XMMWORD[rdi],xmm1 @@ -2092,19 +2092,19 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 - +; Shift the stream left movdqa xmm1,xmm5 movdqa xmm5,xmm9 movdqa xmm9,xmm13 @@ -2116,12 +2116,12 @@ $L$SEH_end_chacha20_poly1305_open_sse41: - - - - - - +; ############################################################################### +; ############################################################################### +; void chacha20_poly1305_seal(uint8_t *out_ciphertext, const uint8_t *plaintext, +; size_t plaintext_len, const uint8_t *ad, +; size_t ad_len, +; union chacha20_poly1305_seal_data *data); global chacha20_poly1305_seal_sse41 ALIGN 64 @@ -2152,8 +2152,8 @@ push r15 - - +; We write the calculated authenticator back to keyp at the end, so save +; the pointer on the stack too. push r9 sub rsp,288 + 160 + 32 @@ -2172,7 +2172,7 @@ movaps XMMWORD[(128+0)+rbp],xmm14 movaps XMMWORD[(144+0)+rbp],xmm15 - mov rbx,QWORD[56+r9] + mov rbx,QWORD[56+r9] ; extra_in_len add rbx,rdx mov QWORD[((0+160+32))+rbp],r8 mov QWORD[((8+160+32))+rbp],rbx @@ -2180,7 +2180,7 @@ cmp rbx,128 jbe NEAR $L$seal_sse_128 - +; For longer buffers, prepare the poly key + some stream movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqu xmm4,XMMWORD[r9] movdqu xmm8,XMMWORD[16+r9] @@ -2201,7 +2201,7 @@ paddd xmm12,XMMWORD[$L$sse_inc] movdqa xmm13,xmm12 paddd xmm12,XMMWORD[$L$sse_inc] - +; Store on stack movdqa XMMWORD[(160+48)+rbp],xmm4 movdqa XMMWORD[(160+64)+rbp],xmm8 movdqa XMMWORD[(160+96)+rbp],xmm12 @@ -2414,11 +2414,11 @@ paddd xmm8,XMMWORD[((160+64))+rbp] paddd xmm12,XMMWORD[((160+96))+rbp] - +; Clamp and store the key pand xmm3,XMMWORD[$L$clamp] movdqa XMMWORD[(160+0)+rbp],xmm3 movdqa XMMWORD[(160+16)+rbp],xmm7 - +; Hash mov r8,r8 call poly_hash_ad_internal movdqu xmm3,XMMWORD[((0 + 0))+rsi] @@ -2635,14 +2635,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -2763,14 +2763,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -2871,7 +2871,7 @@ ja NEAR $L$seal_sse_tail_192 cmp rbx,4*16 ja NEAR $L$seal_sse_tail_128 - +; ############################################################################## $L$seal_sse_tail_64: movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] @@ -2910,14 +2910,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -2995,14 +2995,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3018,7 +3018,7 @@ paddd xmm12,XMMWORD[((160+96))+rbp] jmp NEAR $L$seal_sse_128_tail_xor - +; ############################################################################## $L$seal_sse_tail_128: movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] @@ -3063,14 +3063,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3148,14 +3148,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3232,7 +3232,7 @@ sub rbx,4*16 lea rsi,[64+rsi] jmp NEAR $L$seal_sse_128_tail_hash - +; ############################################################################## $L$seal_sse_tail_192: movdqa xmm0,XMMWORD[$L$chacha20_consts] movdqa xmm4,XMMWORD[((160+48))+rbp] @@ -3283,14 +3283,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3389,14 +3389,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3509,7 +3509,7 @@ mov rcx,8*16 sub rbx,8*16 lea rsi,[128+rsi] - +; ############################################################################## $L$seal_sse_128_tail_hash: cmp rcx,16 jb NEAR $L$seal_sse_128_tail_xor @@ -3542,14 +3542,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3562,11 +3562,11 @@ cmp rbx,16 jb NEAR $L$seal_sse_tail_16 sub rbx,16 - +; Load for decryption movdqu xmm3,XMMWORD[rsi] pxor xmm0,xmm3 movdqu XMMWORD[rdi],xmm0 - +; Then hash add r10,QWORD[rdi] adc r11,QWORD[8+rdi] adc r12,1 @@ -3598,19 +3598,19 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 - +; Shift the stream left movdqa xmm0,xmm4 movdqa xmm4,xmm8 movdqa xmm8,xmm12 @@ -3623,7 +3623,7 @@ $L$seal_sse_tail_16: test rbx,rbx jz NEAR $L$process_blocks_of_extra_in - +; We can only load the PT one byte at a time to avoid buffer overread mov r8,rbx mov rcx,rbx lea rsi,[((-1))+rbx*1+rsi] @@ -3635,10 +3635,10 @@ dec rcx jne NEAR $L$seal_sse_tail_16_compose - +; XOR the keystream with the plaintext. pxor xmm15,xmm0 - +; Write ciphertext out, byte-by-byte. mov rcx,rbx movdqu xmm0,xmm15 $L$seal_sse_tail_16_extract: @@ -3648,42 +3648,42 @@ sub rcx,1 jnz NEAR $L$seal_sse_tail_16_extract - - - - - - - +; %xmm15 contains the final (partial, non-empty) block of ciphertext which +; needs to be fed into the Poly1305 state. The right-most %rbx bytes of it +; are valid. We need to fill it with extra_in bytes until full, or until we +; run out of bytes. +; +; %r9 points to the tag output, which is actually a struct with the +; extra_in pointer and length at offset 48. mov r9,QWORD[((288 + 160 + 32))+rsp] - mov r14,QWORD[56+r9] - mov r13,QWORD[48+r9] + mov r14,QWORD[56+r9] ; extra_in_len + mov r13,QWORD[48+r9] ; extra_in test r14,r14 - jz NEAR $L$process_partial_block + jz NEAR $L$process_partial_block ; Common case: no bytes of extra_in mov r15,16 - sub r15,rbx - cmp r14,r15 - + sub r15,rbx ; 16-%rbx is the number of bytes that fit into %xmm15. + cmp r14,r15 ; if extra_in_len < 16-%rbx, only copy extra_in_len +; (note that AT&T syntax reverses the arguments) jge NEAR $L$load_extra_in mov r15,r14 $L$load_extra_in: - - +; %r15 contains the number of bytes of extra_in (pointed to by %r13) to load +; into %xmm15. They are loaded in reverse order. lea rsi,[((-1))+r15*1+r13] - - +; Update extra_in and extra_in_len to reflect the bytes that are about to +; be read. add r13,r15 sub r14,r15 mov QWORD[48+r9],r13 mov QWORD[56+r9],r14 - - +; Update %r8, which is used to select the mask later on, to reflect the +; extra bytes about to be added. add r8,r15 - +; Load %r15 bytes of extra_in into %xmm11. pxor xmm11,xmm11 $L$load_extra_load_loop: pslldq xmm11,1 @@ -3692,9 +3692,9 @@ sub r15,1 jnz NEAR $L$load_extra_load_loop - - - +; Shift %xmm11 up the length of the remainder from the main encryption. Sadly, +; the shift for an XMM register has to be a constant, thus we loop to do +; this. mov r15,rbx $L$load_extra_shift_loop: @@ -3702,18 +3702,18 @@ sub r15,1 jnz NEAR $L$load_extra_shift_loop - - - +; Mask %xmm15 (the remainder from the main encryption) so that superfluous +; bytes are zero. This means that the non-zero bytes in %xmm11 and %xmm15 are +; disjoint and so we can merge them with an OR. lea r15,[$L$and_masks] shl rbx,4 pand xmm15,XMMWORD[((-16))+rbx*1+r15] - +; Merge %xmm11 into %xmm15, forming the remainder block. por xmm15,xmm11 - - +; The block of ciphertext + extra_in is ready to be included in the +; Poly1305 state. movq r13,xmm15 pextrq r14,xmm15,1 add r10,r13 @@ -3745,26 +3745,26 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 $L$process_blocks_of_extra_in: - +; There may be additional bytes of extra_in to process. mov r9,QWORD[((288+32+160 ))+rsp] - mov rsi,QWORD[48+r9] - mov r8,QWORD[56+r9] + mov rsi,QWORD[48+r9] ; extra_in + mov r8,QWORD[56+r9] ; extra_in_len mov rcx,r8 - shr r8,4 + shr r8,4 ; number of blocks $L$process_extra_hash_loop: jz NEAR process_extra_in_trailer @@ -3797,14 +3797,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3813,7 +3813,7 @@ sub r8,1 jmp NEAR $L$process_extra_hash_loop process_extra_in_trailer: - and rcx,15 + and rcx,15 ; remaining num bytes (<16) of extra_in mov rbx,rcx jz NEAR $L$do_length_block lea rsi,[((-1))+rcx*1+rsi] @@ -3826,7 +3826,7 @@ jnz NEAR $L$process_extra_in_trailer_load $L$process_partial_block: - +; %xmm15 contains %rbx bytes of data to be fed into Poly1305. %rbx != 0 lea r15,[$L$and_masks] shl rbx,4 pand xmm15,XMMWORD[((-16))+rbx*1+r15] @@ -3861,14 +3861,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -3904,19 +3904,19 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 - +; Final reduce mov r13,r10 mov r14,r11 mov r15,r12 @@ -3926,7 +3926,7 @@ cmovc r10,r13 cmovc r11,r14 cmovc r12,r15 - +; Add in s part of the key add r10,QWORD[((0+160+16))+rbp] adc r11,QWORD[((8+160+16))+rbp] @@ -3944,7 +3944,7 @@ add rsp,288 + 160 + 32 - +; The tag replaces the key on return pop r9 mov QWORD[r9],r10 @@ -3964,7 +3964,7 @@ mov rdi,QWORD[8+rsp] ;WIN64 epilogue mov rsi,QWORD[16+rsp] ret - +; ############################################################################### $L$seal_sse_128: movdqu xmm0,XMMWORD[$L$chacha20_consts] @@ -4127,18 +4127,18 @@ paddd xmm12,xmm15 paddd xmm15,XMMWORD[$L$sse_inc] paddd xmm13,xmm15 - +; Clamp and store the key pand xmm2,XMMWORD[$L$clamp] movdqa XMMWORD[(160+0)+rbp],xmm2 movdqa XMMWORD[(160+16)+rbp],xmm6 - +; Hash mov r8,r8 call poly_hash_ad_internal jmp NEAR $L$seal_sse_128_tail_xor $L$SEH_end_chacha20_poly1305_seal_sse41: - +; ############################################################################## global chacha20_poly1305_open_avx2 ALIGN 64 @@ -4169,8 +4169,8 @@ push r15 - - +; We write the calculated authenticator back to keyp at the end, so save +; the pointer on the stack too. push r9 sub rsp,288 + 160 + 32 @@ -4257,16 +4257,16 @@ vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp] vperm2i128 ymm3,ymm4,ymm0,0x02 - +; Clamp and store key vpand ymm3,ymm3,YMMWORD[$L$clamp] vmovdqa YMMWORD[(160+0)+rbp],ymm3 - +; Stream for the first 64 bytes vperm2i128 ymm0,ymm4,ymm0,0x13 vperm2i128 ymm4,ymm12,ymm8,0x13 - +; Hash AD + first 64 bytes mov r8,r8 call poly_hash_ad_internal - +; Hash first 64 bytes xor rcx,rcx $L$open_avx2_init_hash: add r10,QWORD[((0+0))+rcx*1+rsi] @@ -4298,14 +4298,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4313,17 +4313,17 @@ add rcx,16 cmp rcx,2*32 jne NEAR $L$open_avx2_init_hash - +; Decrypt first 64 bytes vpxor ymm0,ymm0,YMMWORD[rsi] vpxor ymm4,ymm4,YMMWORD[32+rsi] - +; Store first 64 bytes of decrypted data vmovdqu YMMWORD[rdi],ymm0 vmovdqu YMMWORD[32+rdi],ymm4 lea rsi,[64+rsi] lea rdi,[64+rdi] sub rbx,2*32 $L$open_avx2_main_loop: - +; Hash and decrypt 512 bytes each iteration cmp rbx,16*32 jb NEAR $L$open_avx2_main_loop_done vmovdqa ymm0,YMMWORD[$L$chacha20_consts] @@ -4413,14 +4413,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4508,14 +4508,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4603,14 +4603,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4679,14 +4679,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4743,14 +4743,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4793,7 +4793,7 @@ mov rcx,rbx and rcx,-16 test rcx,rcx - je NEAR $L$open_avx2_tail_128_rounds + je NEAR $L$open_avx2_tail_128_rounds ; Have nothing to hash $L$open_avx2_tail_128_rounds_and_x1hash: add r10,QWORD[((0+0))+r8*1+rsi] adc r11,QWORD[((8+0))+r8*1+rsi] @@ -4824,14 +4824,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -4892,7 +4892,7 @@ vmovdqa ymm8,ymm3 jmp NEAR $L$open_avx2_tail_128_xor - +; ############################################################################## $L$open_avx2_tail_256: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -4938,14 +4938,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5084,14 +5084,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5129,7 +5129,7 @@ lea rdi,[128+rdi] sub rbx,4*32 jmp NEAR $L$open_avx2_tail_128_xor - +; ############################################################################## $L$open_avx2_tail_384: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -5181,14 +5181,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5281,14 +5281,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5387,14 +5387,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5448,7 +5448,7 @@ lea rdi,[256+rdi] sub rbx,8*32 jmp NEAR $L$open_avx2_tail_128_xor - +; ############################################################################## $L$open_avx2_tail_512: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -5504,14 +5504,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5577,14 +5577,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5658,14 +5658,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5784,14 +5784,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -5882,7 +5882,7 @@ vmovdqa xmm1,xmm0 jb NEAR $L$open_avx2_exit sub rbx,16 - +; load for decryption vpxor xmm1,xmm0,XMMWORD[rsi] vmovdqu XMMWORD[rdi],xmm1 lea rsi,[16+rsi] @@ -5892,7 +5892,7 @@ $L$open_avx2_exit: vzeroupper jmp NEAR $L$open_sse_tail_16 - +; ############################################################################## $L$open_avx2_192: vmovdqa ymm1,ymm0 vmovdqa ymm2,ymm0 @@ -5993,10 +5993,10 @@ vpaddd ymm12,ymm12,ymm11 vpaddd ymm13,ymm13,ymm15 vperm2i128 ymm3,ymm4,ymm0,0x02 - +; Clamp and store the key vpand ymm3,ymm3,YMMWORD[$L$clamp] vmovdqa YMMWORD[(160+0)+rbp],ymm3 - +; Stream for up to 192 bytes vperm2i128 ymm0,ymm4,ymm0,0x13 vperm2i128 ymm4,ymm12,ymm8,0x13 vperm2i128 ymm8,ymm5,ymm1,0x02 @@ -6039,14 +6039,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -6079,24 +6079,24 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 - +; Load + decrypt vpxor ymm0,ymm0,YMMWORD[rsi] vmovdqu YMMWORD[rdi],ymm0 lea rsi,[32+rsi] lea rdi,[32+rdi] - +; Shift stream vmovdqa ymm0,ymm4 vmovdqa ymm4,ymm8 vmovdqa ymm8,ymm12 @@ -6141,14 +6141,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -6161,7 +6161,7 @@ $L$open_avx2_short_tail_32_exit: vzeroupper jmp NEAR $L$open_sse_tail_16 - +; ############################################################################## $L$open_avx2_320: vmovdqa ymm1,ymm0 vmovdqa ymm2,ymm0 @@ -6308,10 +6308,10 @@ vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp] vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp] vperm2i128 ymm3,ymm4,ymm0,0x02 - +; Clamp and store the key vpand ymm3,ymm3,YMMWORD[$L$clamp] vmovdqa YMMWORD[(160+0)+rbp],ymm3 - +; Stream for up to 320 bytes vperm2i128 ymm0,ymm4,ymm0,0x13 vperm2i128 ymm4,ymm12,ymm8,0x13 vperm2i128 ymm8,ymm5,ymm1,0x02 @@ -6325,8 +6325,8 @@ jmp NEAR $L$open_avx2_short $L$SEH_end_chacha20_poly1305_open_avx2: - - +; ############################################################################## +; ############################################################################## global chacha20_poly1305_seal_avx2 ALIGN 64 @@ -6357,8 +6357,8 @@ push r15 - - +; We write the calculated authenticator back to keyp at the end, so save +; the pointer on the stack too. push r9 sub rsp,288 + 160 + 32 @@ -6377,7 +6377,7 @@ movaps XMMWORD[(128+0)+rbp],xmm14 movaps XMMWORD[(144+0)+rbp],xmm15 - mov rbx,QWORD[56+r9] + mov rbx,QWORD[56+r9] ; extra_in_len add rbx,rdx mov QWORD[((0+160+32))+rbp],r8 mov QWORD[((8+160+32))+rbp],rbx @@ -6605,7 +6605,7 @@ vmovdqa YMMWORD[(160+0)+rbp],ymm15 mov r8,r8 call poly_hash_ad_internal - +; Safely store 320 bytes (otherwise would handle with optimized call) vpxor ymm3,ymm3,YMMWORD[rsi] vpxor ymm11,ymm11,YMMWORD[32+rsi] vmovdqu YMMWORD[rdi],ymm3 @@ -6985,14 +6985,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7082,14 +7082,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7177,14 +7177,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7232,14 +7232,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7265,14 +7265,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7356,14 +7356,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7389,14 +7389,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7411,7 +7411,7 @@ ja NEAR $L$seal_avx2_tail_384 cmp rbx,4*32 ja NEAR $L$seal_avx2_tail_256 - +; ############################################################################## $L$seal_avx2_tail_128: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -7443,14 +7443,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7498,14 +7498,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7550,14 +7550,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7578,7 +7578,7 @@ vmovdqa ymm8,ymm3 jmp NEAR $L$seal_avx2_short_loop - +; ############################################################################## $L$seal_avx2_tail_256: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -7622,14 +7622,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7703,14 +7703,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7781,14 +7781,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7828,7 +7828,7 @@ lea rsi,[128+rsi] sub rbx,4*32 jmp NEAR $L$seal_avx2_short_hash_remainder - +; ############################################################################## $L$seal_avx2_tail_384: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -7877,14 +7877,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -7958,14 +7958,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8036,14 +8036,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8137,7 +8137,7 @@ lea rsi,[256+rsi] sub rbx,8*32 jmp NEAR $L$seal_avx2_short_hash_remainder - +; ############################################################################## $L$seal_avx2_tail_512: vmovdqa ymm0,YMMWORD[$L$chacha20_consts] vmovdqa ymm4,YMMWORD[((160+64))+rbp] @@ -8184,14 +8184,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8320,14 +8320,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8453,14 +8453,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8536,7 +8536,7 @@ lea rsi,[384+rsi] sub rbx,12*32 jmp NEAR $L$seal_avx2_short_hash_remainder - +; ############################################################################### $L$seal_avx2_320: vmovdqa ymm1,ymm0 vmovdqa ymm2,ymm0 @@ -8683,10 +8683,10 @@ vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp] vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp] vperm2i128 ymm3,ymm4,ymm0,0x02 - +; Clamp and store the key vpand ymm3,ymm3,YMMWORD[$L$clamp] vmovdqa YMMWORD[(160+0)+rbp],ymm3 - +; Stream for up to 320 bytes vperm2i128 ymm0,ymm4,ymm0,0x13 vperm2i128 ymm4,ymm12,ymm8,0x13 vperm2i128 ymm8,ymm5,ymm1,0x02 @@ -8698,7 +8698,7 @@ vperm2i128 ymm2,ymm6,ymm2,0x13 vperm2i128 ymm6,ymm14,ymm10,0x13 jmp NEAR $L$seal_avx2_short - +; ############################################################################### $L$seal_avx2_192: vmovdqa ymm1,ymm0 vmovdqa ymm2,ymm0 @@ -8799,10 +8799,10 @@ vpaddd ymm12,ymm12,ymm11 vpaddd ymm13,ymm13,ymm15 vperm2i128 ymm3,ymm4,ymm0,0x02 - +; Clamp and store the key vpand ymm3,ymm3,YMMWORD[$L$clamp] vmovdqa YMMWORD[(160+0)+rbp],ymm3 - +; Stream for up to 192 bytes vperm2i128 ymm0,ymm4,ymm0,0x13 vperm2i128 ymm4,ymm12,ymm8,0x13 vperm2i128 ymm8,ymm5,ymm1,0x02 @@ -8845,14 +8845,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8864,11 +8864,11 @@ cmp rbx,32 jb NEAR $L$seal_avx2_short_tail sub rbx,32 - +; Encrypt vpxor ymm0,ymm0,YMMWORD[rsi] vmovdqu YMMWORD[rdi],ymm0 lea rsi,[32+rsi] - +; Load + hash add r10,QWORD[((0+0))+rdi] adc r11,QWORD[((8+0))+rdi] adc r12,1 @@ -8898,14 +8898,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 @@ -8938,20 +8938,20 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0 lea rdi,[32+rdi] - +; Shift stream vmovdqa ymm0,ymm4 vmovdqa ymm4,ymm8 vmovdqa ymm8,ymm12 @@ -8998,14 +8998,14 @@ mov r10,r13 mov r11,r14 mov r12,r15 - and r12,3 + and r12,3 ; At this point acc2 is 2 bits at most (value of 3) mov r13,r15 and r13,-4 mov r14,r9 shrd r15,r9,2 shr r9,2 add r15,r13 - adc r9,r14 + adc r9,r14 ; No carry out since t3 is 61 bits and t1 is 63 bits add r10,r15 adc r11,r9 adc r12,0
diff --git a/gen/crypto/md5-x86_64-apple.S b/gen/crypto/md5-x86_64-apple.S index e4c0241..cebbe92 100644 --- a/gen/crypto/md5-x86_64-apple.S +++ b/gen/crypto/md5-x86_64-apple.S
@@ -25,651 +25,651 @@ L$prologue: +// rdi = arg #1 (ctx, MD5_CTX pointer) +// rsi = arg #2 (ptr, data pointer) +// rdx = arg #3 (nbr, number of 16-word blocks to process) + movq %rdi,%rbp // rbp = ctx + shlq $6,%rdx // rdx = nbr in bytes + leaq (%rsi,%rdx,1),%rdi // rdi = end + movl 0(%rbp),%eax // eax = ctx->A + movl 4(%rbp),%ebx // ebx = ctx->B + movl 8(%rbp),%ecx // ecx = ctx->C + movl 12(%rbp),%edx // edx = ctx->D +// end is 'rdi' +// ptr is 'rsi' +// A is 'eax' +// B is 'ebx' +// C is 'ecx' +// D is 'edx' + cmpq %rdi,%rsi // cmp end with ptr + je L$end // jmp if ptr == end - - movq %rdi,%rbp - shlq $6,%rdx - leaq (%rsi,%rdx,1),%rdi - movl 0(%rbp),%eax - movl 4(%rbp),%ebx - movl 8(%rbp),%ecx - movl 12(%rbp),%edx - - - - - - - - cmpq %rdi,%rsi - je L$end - - -L$loop: +// BEGIN of loop over 16-word blocks +L$loop: // save old values of A, B, C, D movl %eax,%r8d movl %ebx,%r9d movl %ecx,%r14d movl %edx,%r15d - movl 0(%rsi),%r10d - movl %edx,%r11d - xorl %ecx,%r11d - leal -680876936(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 4(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal -389564586(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 8(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal 606105819(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 12(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal -1044525330(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 16(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - xorl %ecx,%r11d - leal -176418897(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 20(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal 1200080426(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 24(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal -1473231341(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 28(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal -45705983(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 32(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - xorl %ecx,%r11d - leal 1770035416(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 36(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal -1958414417(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 40(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal -42063(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 44(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal -1990404162(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 48(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - xorl %ecx,%r11d - leal 1804603682(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 52(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal -40341101(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 56(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal -1502002290(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 60(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal 1236535329(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 0(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - movl 4(%rsi),%r10d - movl %edx,%r11d - movl %edx,%r12d - notl %r11d - leal -165796510(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 24(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal -1069501632(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 44(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal 643717713(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 0(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal -373897302(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 20(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - notl %r11d - leal -701558691(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 40(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal 38016083(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 60(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal -660478335(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 16(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal -405537848(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 36(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - notl %r11d - leal 568446438(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 56(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal -1019803690(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 12(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal -187363961(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 32(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal 1163531501(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 52(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - notl %r11d - leal -1444681467(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 8(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal -51403784(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 28(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal 1735328473(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 48(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal -1926607734(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 0(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - movl 20(%rsi),%r10d - movl %ecx,%r11d - leal -378558(%rax,%r10,1),%eax - movl 32(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal -2022574463(%rdx,%r10,1),%edx - movl 44(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal 1839030562(%rcx,%r10,1),%ecx - movl 56(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal -35309556(%rbx,%r10,1),%ebx - movl 4(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - leal -1530992060(%rax,%r10,1),%eax - movl 16(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal 1272893353(%rdx,%r10,1),%edx - movl 28(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal -155497632(%rcx,%r10,1),%ecx - movl 40(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal -1094730640(%rbx,%r10,1),%ebx - movl 52(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - leal 681279174(%rax,%r10,1),%eax - movl 0(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal -358537222(%rdx,%r10,1),%edx - movl 12(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal -722521979(%rcx,%r10,1),%ecx - movl 24(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal 76029189(%rbx,%r10,1),%ebx - movl 36(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - leal -640364487(%rax,%r10,1),%eax - movl 48(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal -421815835(%rdx,%r10,1),%edx - movl 60(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal 530742520(%rcx,%r10,1),%ecx - movl 8(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal -995338651(%rbx,%r10,1),%ebx - movl 0(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - movl 0(%rsi),%r10d + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + movl %edx,%r11d // (NEXT STEP) z' = %edx + xorl %ecx,%r11d // y ^ ... + leal -680876936(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 4(%rsi),%r10d // (NEXT STEP) X[1] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal -389564586(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 8(%rsi),%r10d // (NEXT STEP) X[2] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal 606105819(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 12(%rsi),%r10d // (NEXT STEP) X[3] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal -1044525330(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 16(%rsi),%r10d // (NEXT STEP) X[4] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + xorl %ecx,%r11d // y ^ ... + leal -176418897(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 20(%rsi),%r10d // (NEXT STEP) X[5] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal 1200080426(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 24(%rsi),%r10d // (NEXT STEP) X[6] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal -1473231341(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 28(%rsi),%r10d // (NEXT STEP) X[7] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal -45705983(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 32(%rsi),%r10d // (NEXT STEP) X[8] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + xorl %ecx,%r11d // y ^ ... + leal 1770035416(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 36(%rsi),%r10d // (NEXT STEP) X[9] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal -1958414417(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 40(%rsi),%r10d // (NEXT STEP) X[10] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal -42063(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 44(%rsi),%r10d // (NEXT STEP) X[11] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal -1990404162(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 48(%rsi),%r10d // (NEXT STEP) X[12] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + xorl %ecx,%r11d // y ^ ... + leal 1804603682(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 52(%rsi),%r10d // (NEXT STEP) X[13] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal -40341101(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 56(%rsi),%r10d // (NEXT STEP) X[14] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal -1502002290(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 60(%rsi),%r10d // (NEXT STEP) X[15] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal 1236535329(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + movl 4(%rsi),%r10d // (NEXT STEP) X[1] + movl %edx,%r11d // (NEXT STEP) z' = %edx + movl %edx,%r12d // (NEXT STEP) z' = %edx + notl %r11d // not z + leal -165796510(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 24(%rsi),%r10d // (NEXT STEP) X[6] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal -1069501632(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 44(%rsi),%r10d // (NEXT STEP) X[11] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal 643717713(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal -373897302(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 20(%rsi),%r10d // (NEXT STEP) X[5] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + notl %r11d // not z + leal -701558691(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 40(%rsi),%r10d // (NEXT STEP) X[10] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal 38016083(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 60(%rsi),%r10d // (NEXT STEP) X[15] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal -660478335(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 16(%rsi),%r10d // (NEXT STEP) X[4] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal -405537848(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 36(%rsi),%r10d // (NEXT STEP) X[9] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + notl %r11d // not z + leal 568446438(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 56(%rsi),%r10d // (NEXT STEP) X[14] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal -1019803690(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 12(%rsi),%r10d // (NEXT STEP) X[3] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal -187363961(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 32(%rsi),%r10d // (NEXT STEP) X[8] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal 1163531501(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 52(%rsi),%r10d // (NEXT STEP) X[13] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + notl %r11d // not z + leal -1444681467(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 8(%rsi),%r10d // (NEXT STEP) X[2] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal -51403784(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 28(%rsi),%r10d // (NEXT STEP) X[7] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal 1735328473(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 48(%rsi),%r10d // (NEXT STEP) X[12] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal -1926607734(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + movl 20(%rsi),%r10d // (NEXT STEP) X[5] + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + leal -378558(%rax,%r10,1),%eax // Const + dst + ... + movl 32(%rsi),%r10d // (NEXT STEP) X[8] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal -2022574463(%rdx,%r10,1),%edx // Const + dst + ... + movl 44(%rsi),%r10d // (NEXT STEP) X[11] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal 1839030562(%rcx,%r10,1),%ecx // Const + dst + ... + movl 56(%rsi),%r10d // (NEXT STEP) X[14] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal -35309556(%rbx,%r10,1),%ebx // Const + dst + ... + movl 4(%rsi),%r10d // (NEXT STEP) X[1] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + leal -1530992060(%rax,%r10,1),%eax // Const + dst + ... + movl 16(%rsi),%r10d // (NEXT STEP) X[4] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal 1272893353(%rdx,%r10,1),%edx // Const + dst + ... + movl 28(%rsi),%r10d // (NEXT STEP) X[7] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal -155497632(%rcx,%r10,1),%ecx // Const + dst + ... + movl 40(%rsi),%r10d // (NEXT STEP) X[10] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal -1094730640(%rbx,%r10,1),%ebx // Const + dst + ... + movl 52(%rsi),%r10d // (NEXT STEP) X[13] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + leal 681279174(%rax,%r10,1),%eax // Const + dst + ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal -358537222(%rdx,%r10,1),%edx // Const + dst + ... + movl 12(%rsi),%r10d // (NEXT STEP) X[3] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal -722521979(%rcx,%r10,1),%ecx // Const + dst + ... + movl 24(%rsi),%r10d // (NEXT STEP) X[6] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal 76029189(%rbx,%r10,1),%ebx // Const + dst + ... + movl 36(%rsi),%r10d // (NEXT STEP) X[9] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + leal -640364487(%rax,%r10,1),%eax // Const + dst + ... + movl 48(%rsi),%r10d // (NEXT STEP) X[12] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal -421815835(%rdx,%r10,1),%edx // Const + dst + ... + movl 60(%rsi),%r10d // (NEXT STEP) X[15] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal 530742520(%rcx,%r10,1),%ecx // Const + dst + ... + movl 8(%rsi),%r10d // (NEXT STEP) X[2] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal -995338651(%rbx,%r10,1),%ebx // Const + dst + ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + movl 0(%rsi),%r10d // (NEXT STEP) X[0] movl $0xffffffff,%r11d - xorl %edx,%r11d - leal -198630844(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 28(%rsi),%r10d + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + leal -198630844(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 28(%rsi),%r10d // (NEXT STEP) X[7] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal 1126891415(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 56(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal 1126891415(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 56(%rsi),%r10d // (NEXT STEP) X[14] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal -1416354905(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 20(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal -1416354905(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 20(%rsi),%r10d // (NEXT STEP) X[5] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal -57434055(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 48(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal -57434055(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 48(%rsi),%r10d // (NEXT STEP) X[12] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - leal 1700485571(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 12(%rsi),%r10d + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x + leal 1700485571(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 12(%rsi),%r10d // (NEXT STEP) X[3] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal -1894986606(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 40(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal -1894986606(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 40(%rsi),%r10d // (NEXT STEP) X[10] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal -1051523(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 4(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal -1051523(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 4(%rsi),%r10d // (NEXT STEP) X[1] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal -2054922799(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 32(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal -2054922799(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 32(%rsi),%r10d // (NEXT STEP) X[8] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - leal 1873313359(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 60(%rsi),%r10d + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x + leal 1873313359(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 60(%rsi),%r10d // (NEXT STEP) X[15] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal -30611744(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 24(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal -30611744(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 24(%rsi),%r10d // (NEXT STEP) X[6] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal -1560198380(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 52(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal -1560198380(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 52(%rsi),%r10d // (NEXT STEP) X[13] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal 1309151649(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 16(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal 1309151649(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 16(%rsi),%r10d // (NEXT STEP) X[4] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - leal -145523070(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 44(%rsi),%r10d + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x + leal -145523070(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 44(%rsi),%r10d // (NEXT STEP) X[11] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal -1120210379(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 8(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal -1120210379(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 8(%rsi),%r10d // (NEXT STEP) X[2] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal 718787259(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 36(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal 718787259(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 36(%rsi),%r10d // (NEXT STEP) X[9] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal -343485551(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 0(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal -343485551(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x +// add old values of A, B, C, D addl %r8d,%eax addl %r9d,%ebx addl %r14d,%ecx addl %r15d,%edx - - addq $64,%rsi - cmpq %rdi,%rsi - jb L$loop - +// loop control + addq $64,%rsi // ptr += 64 + cmpq %rdi,%rsi // cmp end with ptr + jb L$loop // jmp if ptr < end +// END of loop over 16-word blocks L$end: - movl %eax,0(%rbp) - movl %ebx,4(%rbp) - movl %ecx,8(%rbp) - movl %edx,12(%rbp) + movl %eax,0(%rbp) // ctx->A = A + movl %ebx,4(%rbp) // ctx->B = B + movl %ecx,8(%rbp) // ctx->C = C + movl %edx,12(%rbp) // ctx->D = D movq (%rsp),%r15
diff --git a/gen/crypto/md5-x86_64-linux.S b/gen/crypto/md5-x86_64-linux.S index 7b93662..e8852c4 100644 --- a/gen/crypto/md5-x86_64-linux.S +++ b/gen/crypto/md5-x86_64-linux.S
@@ -30,651 +30,651 @@ .cfi_offset r15,-48 .Lprologue: +// rdi = arg #1 (ctx, MD5_CTX pointer) +// rsi = arg #2 (ptr, data pointer) +// rdx = arg #3 (nbr, number of 16-word blocks to process) + movq %rdi,%rbp // rbp = ctx + shlq $6,%rdx // rdx = nbr in bytes + leaq (%rsi,%rdx,1),%rdi // rdi = end + movl 0(%rbp),%eax // eax = ctx->A + movl 4(%rbp),%ebx // ebx = ctx->B + movl 8(%rbp),%ecx // ecx = ctx->C + movl 12(%rbp),%edx // edx = ctx->D +// end is 'rdi' +// ptr is 'rsi' +// A is 'eax' +// B is 'ebx' +// C is 'ecx' +// D is 'edx' + cmpq %rdi,%rsi // cmp end with ptr + je .Lend // jmp if ptr == end - - movq %rdi,%rbp - shlq $6,%rdx - leaq (%rsi,%rdx,1),%rdi - movl 0(%rbp),%eax - movl 4(%rbp),%ebx - movl 8(%rbp),%ecx - movl 12(%rbp),%edx - - - - - - - - cmpq %rdi,%rsi - je .Lend - - -.Lloop: +// BEGIN of loop over 16-word blocks +.Lloop: // save old values of A, B, C, D movl %eax,%r8d movl %ebx,%r9d movl %ecx,%r14d movl %edx,%r15d - movl 0(%rsi),%r10d - movl %edx,%r11d - xorl %ecx,%r11d - leal -680876936(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 4(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal -389564586(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 8(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal 606105819(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 12(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal -1044525330(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 16(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - xorl %ecx,%r11d - leal -176418897(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 20(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal 1200080426(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 24(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal -1473231341(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 28(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal -45705983(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 32(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - xorl %ecx,%r11d - leal 1770035416(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 36(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal -1958414417(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 40(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal -42063(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 44(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal -1990404162(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 48(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - xorl %ecx,%r11d - leal 1804603682(%rax,%r10,1),%eax - andl %ebx,%r11d - xorl %edx,%r11d - movl 52(%rsi),%r10d - addl %r11d,%eax - roll $7,%eax - movl %ecx,%r11d - addl %ebx,%eax - xorl %ebx,%r11d - leal -40341101(%rdx,%r10,1),%edx - andl %eax,%r11d - xorl %ecx,%r11d - movl 56(%rsi),%r10d - addl %r11d,%edx - roll $12,%edx - movl %ebx,%r11d - addl %eax,%edx - xorl %eax,%r11d - leal -1502002290(%rcx,%r10,1),%ecx - andl %edx,%r11d - xorl %ebx,%r11d - movl 60(%rsi),%r10d - addl %r11d,%ecx - roll $17,%ecx - movl %eax,%r11d - addl %edx,%ecx - xorl %edx,%r11d - leal 1236535329(%rbx,%r10,1),%ebx - andl %ecx,%r11d - xorl %eax,%r11d - movl 0(%rsi),%r10d - addl %r11d,%ebx - roll $22,%ebx - movl %edx,%r11d - addl %ecx,%ebx - movl 4(%rsi),%r10d - movl %edx,%r11d - movl %edx,%r12d - notl %r11d - leal -165796510(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 24(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal -1069501632(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 44(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal 643717713(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 0(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal -373897302(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 20(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - notl %r11d - leal -701558691(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 40(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal 38016083(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 60(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal -660478335(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 16(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal -405537848(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 36(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - notl %r11d - leal 568446438(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 56(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal -1019803690(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 12(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal -187363961(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 32(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal 1163531501(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 52(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - notl %r11d - leal -1444681467(%rax,%r10,1),%eax - andl %ebx,%r12d - andl %ecx,%r11d - movl 8(%rsi),%r10d - orl %r11d,%r12d - movl %ecx,%r11d - addl %r12d,%eax - movl %ecx,%r12d - roll $5,%eax - addl %ebx,%eax - notl %r11d - leal -51403784(%rdx,%r10,1),%edx - andl %eax,%r12d - andl %ebx,%r11d - movl 28(%rsi),%r10d - orl %r11d,%r12d - movl %ebx,%r11d - addl %r12d,%edx - movl %ebx,%r12d - roll $9,%edx - addl %eax,%edx - notl %r11d - leal 1735328473(%rcx,%r10,1),%ecx - andl %edx,%r12d - andl %eax,%r11d - movl 48(%rsi),%r10d - orl %r11d,%r12d - movl %eax,%r11d - addl %r12d,%ecx - movl %eax,%r12d - roll $14,%ecx - addl %edx,%ecx - notl %r11d - leal -1926607734(%rbx,%r10,1),%ebx - andl %ecx,%r12d - andl %edx,%r11d - movl 0(%rsi),%r10d - orl %r11d,%r12d - movl %edx,%r11d - addl %r12d,%ebx - movl %edx,%r12d - roll $20,%ebx - addl %ecx,%ebx - movl 20(%rsi),%r10d - movl %ecx,%r11d - leal -378558(%rax,%r10,1),%eax - movl 32(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal -2022574463(%rdx,%r10,1),%edx - movl 44(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal 1839030562(%rcx,%r10,1),%ecx - movl 56(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal -35309556(%rbx,%r10,1),%ebx - movl 4(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - leal -1530992060(%rax,%r10,1),%eax - movl 16(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal 1272893353(%rdx,%r10,1),%edx - movl 28(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal -155497632(%rcx,%r10,1),%ecx - movl 40(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal -1094730640(%rbx,%r10,1),%ebx - movl 52(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - leal 681279174(%rax,%r10,1),%eax - movl 0(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal -358537222(%rdx,%r10,1),%edx - movl 12(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal -722521979(%rcx,%r10,1),%ecx - movl 24(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal 76029189(%rbx,%r10,1),%ebx - movl 36(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - leal -640364487(%rax,%r10,1),%eax - movl 48(%rsi),%r10d - xorl %edx,%r11d - xorl %ebx,%r11d - addl %r11d,%eax - roll $4,%eax - movl %ebx,%r11d - addl %ebx,%eax - leal -421815835(%rdx,%r10,1),%edx - movl 60(%rsi),%r10d - xorl %ecx,%r11d - xorl %eax,%r11d - addl %r11d,%edx - roll $11,%edx - movl %eax,%r11d - addl %eax,%edx - leal 530742520(%rcx,%r10,1),%ecx - movl 8(%rsi),%r10d - xorl %ebx,%r11d - xorl %edx,%r11d - addl %r11d,%ecx - roll $16,%ecx - movl %edx,%r11d - addl %edx,%ecx - leal -995338651(%rbx,%r10,1),%ebx - movl 0(%rsi),%r10d - xorl %eax,%r11d - xorl %ecx,%r11d - addl %r11d,%ebx - roll $23,%ebx - movl %ecx,%r11d - addl %ecx,%ebx - movl 0(%rsi),%r10d + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + movl %edx,%r11d // (NEXT STEP) z' = %edx + xorl %ecx,%r11d // y ^ ... + leal -680876936(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 4(%rsi),%r10d // (NEXT STEP) X[1] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal -389564586(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 8(%rsi),%r10d // (NEXT STEP) X[2] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal 606105819(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 12(%rsi),%r10d // (NEXT STEP) X[3] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal -1044525330(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 16(%rsi),%r10d // (NEXT STEP) X[4] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + xorl %ecx,%r11d // y ^ ... + leal -176418897(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 20(%rsi),%r10d // (NEXT STEP) X[5] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal 1200080426(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 24(%rsi),%r10d // (NEXT STEP) X[6] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal -1473231341(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 28(%rsi),%r10d // (NEXT STEP) X[7] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal -45705983(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 32(%rsi),%r10d // (NEXT STEP) X[8] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + xorl %ecx,%r11d // y ^ ... + leal 1770035416(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 36(%rsi),%r10d // (NEXT STEP) X[9] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal -1958414417(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 40(%rsi),%r10d // (NEXT STEP) X[10] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal -42063(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 44(%rsi),%r10d // (NEXT STEP) X[11] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal -1990404162(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 48(%rsi),%r10d // (NEXT STEP) X[12] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + xorl %ecx,%r11d // y ^ ... + leal 1804603682(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r11d // x & ... + xorl %edx,%r11d // z ^ ... + movl 52(%rsi),%r10d // (NEXT STEP) X[13] + addl %r11d,%eax // dst += ... + roll $7,%eax // dst <<< s + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %ebx,%eax // dst += x + xorl %ebx,%r11d // y ^ ... + leal -40341101(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r11d // x & ... + xorl %ecx,%r11d // z ^ ... + movl 56(%rsi),%r10d // (NEXT STEP) X[14] + addl %r11d,%edx // dst += ... + roll $12,%edx // dst <<< s + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %eax,%edx // dst += x + xorl %eax,%r11d // y ^ ... + leal -1502002290(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r11d // x & ... + xorl %ebx,%r11d // z ^ ... + movl 60(%rsi),%r10d // (NEXT STEP) X[15] + addl %r11d,%ecx // dst += ... + roll $17,%ecx // dst <<< s + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %edx,%ecx // dst += x + xorl %edx,%r11d // y ^ ... + leal 1236535329(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r11d // x & ... + xorl %eax,%r11d // z ^ ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + addl %r11d,%ebx // dst += ... + roll $22,%ebx // dst <<< s + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %ecx,%ebx // dst += x + movl 4(%rsi),%r10d // (NEXT STEP) X[1] + movl %edx,%r11d // (NEXT STEP) z' = %edx + movl %edx,%r12d // (NEXT STEP) z' = %edx + notl %r11d // not z + leal -165796510(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 24(%rsi),%r10d // (NEXT STEP) X[6] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal -1069501632(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 44(%rsi),%r10d // (NEXT STEP) X[11] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal 643717713(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal -373897302(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 20(%rsi),%r10d // (NEXT STEP) X[5] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + notl %r11d // not z + leal -701558691(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 40(%rsi),%r10d // (NEXT STEP) X[10] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal 38016083(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 60(%rsi),%r10d // (NEXT STEP) X[15] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal -660478335(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 16(%rsi),%r10d // (NEXT STEP) X[4] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal -405537848(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 36(%rsi),%r10d // (NEXT STEP) X[9] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + notl %r11d // not z + leal 568446438(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 56(%rsi),%r10d // (NEXT STEP) X[14] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal -1019803690(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 12(%rsi),%r10d // (NEXT STEP) X[3] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal -187363961(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 32(%rsi),%r10d // (NEXT STEP) X[8] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal 1163531501(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 52(%rsi),%r10d // (NEXT STEP) X[13] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + notl %r11d // not z + leal -1444681467(%rax,%r10,1),%eax // Const + dst + ... + andl %ebx,%r12d // x & z + andl %ecx,%r11d // y & (not z) + movl 8(%rsi),%r10d // (NEXT STEP) X[2] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ecx,%r11d // (NEXT STEP) z' = %ecx + addl %r12d,%eax // dst += ... + movl %ecx,%r12d // (NEXT STEP) z' = %ecx + roll $5,%eax // dst <<< s + addl %ebx,%eax // dst += x + notl %r11d // not z + leal -51403784(%rdx,%r10,1),%edx // Const + dst + ... + andl %eax,%r12d // x & z + andl %ebx,%r11d // y & (not z) + movl 28(%rsi),%r10d // (NEXT STEP) X[7] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %ebx,%r11d // (NEXT STEP) z' = %ebx + addl %r12d,%edx // dst += ... + movl %ebx,%r12d // (NEXT STEP) z' = %ebx + roll $9,%edx // dst <<< s + addl %eax,%edx // dst += x + notl %r11d // not z + leal 1735328473(%rcx,%r10,1),%ecx // Const + dst + ... + andl %edx,%r12d // x & z + andl %eax,%r11d // y & (not z) + movl 48(%rsi),%r10d // (NEXT STEP) X[12] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %eax,%r11d // (NEXT STEP) z' = %eax + addl %r12d,%ecx // dst += ... + movl %eax,%r12d // (NEXT STEP) z' = %eax + roll $14,%ecx // dst <<< s + addl %edx,%ecx // dst += x + notl %r11d // not z + leal -1926607734(%rbx,%r10,1),%ebx // Const + dst + ... + andl %ecx,%r12d // x & z + andl %edx,%r11d // y & (not z) + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + orl %r11d,%r12d // (y & (not z)) | (x & z) + movl %edx,%r11d // (NEXT STEP) z' = %edx + addl %r12d,%ebx // dst += ... + movl %edx,%r12d // (NEXT STEP) z' = %edx + roll $20,%ebx // dst <<< s + addl %ecx,%ebx // dst += x + movl 20(%rsi),%r10d // (NEXT STEP) X[5] + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + leal -378558(%rax,%r10,1),%eax // Const + dst + ... + movl 32(%rsi),%r10d // (NEXT STEP) X[8] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal -2022574463(%rdx,%r10,1),%edx // Const + dst + ... + movl 44(%rsi),%r10d // (NEXT STEP) X[11] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal 1839030562(%rcx,%r10,1),%ecx // Const + dst + ... + movl 56(%rsi),%r10d // (NEXT STEP) X[14] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal -35309556(%rbx,%r10,1),%ebx // Const + dst + ... + movl 4(%rsi),%r10d // (NEXT STEP) X[1] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + leal -1530992060(%rax,%r10,1),%eax // Const + dst + ... + movl 16(%rsi),%r10d // (NEXT STEP) X[4] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal 1272893353(%rdx,%r10,1),%edx // Const + dst + ... + movl 28(%rsi),%r10d // (NEXT STEP) X[7] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal -155497632(%rcx,%r10,1),%ecx // Const + dst + ... + movl 40(%rsi),%r10d // (NEXT STEP) X[10] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal -1094730640(%rbx,%r10,1),%ebx // Const + dst + ... + movl 52(%rsi),%r10d // (NEXT STEP) X[13] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + leal 681279174(%rax,%r10,1),%eax // Const + dst + ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal -358537222(%rdx,%r10,1),%edx // Const + dst + ... + movl 12(%rsi),%r10d // (NEXT STEP) X[3] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal -722521979(%rcx,%r10,1),%ecx // Const + dst + ... + movl 24(%rsi),%r10d // (NEXT STEP) X[6] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal 76029189(%rbx,%r10,1),%ebx // Const + dst + ... + movl 36(%rsi),%r10d // (NEXT STEP) X[9] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + leal -640364487(%rax,%r10,1),%eax // Const + dst + ... + movl 48(%rsi),%r10d // (NEXT STEP) X[12] + xorl %edx,%r11d // z ^ ... + xorl %ebx,%r11d // x ^ ... + addl %r11d,%eax // dst += ... + roll $4,%eax // dst <<< s + movl %ebx,%r11d // (NEXT STEP) y' = %ebx + addl %ebx,%eax // dst += x + leal -421815835(%rdx,%r10,1),%edx // Const + dst + ... + movl 60(%rsi),%r10d // (NEXT STEP) X[15] + xorl %ecx,%r11d // z ^ ... + xorl %eax,%r11d // x ^ ... + addl %r11d,%edx // dst += ... + roll $11,%edx // dst <<< s + movl %eax,%r11d // (NEXT STEP) y' = %eax + addl %eax,%edx // dst += x + leal 530742520(%rcx,%r10,1),%ecx // Const + dst + ... + movl 8(%rsi),%r10d // (NEXT STEP) X[2] + xorl %ebx,%r11d // z ^ ... + xorl %edx,%r11d // x ^ ... + addl %r11d,%ecx // dst += ... + roll $16,%ecx // dst <<< s + movl %edx,%r11d // (NEXT STEP) y' = %edx + addl %edx,%ecx // dst += x + leal -995338651(%rbx,%r10,1),%ebx // Const + dst + ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] + xorl %eax,%r11d // z ^ ... + xorl %ecx,%r11d // x ^ ... + addl %r11d,%ebx // dst += ... + roll $23,%ebx // dst <<< s + movl %ecx,%r11d // (NEXT STEP) y' = %ecx + addl %ecx,%ebx // dst += x + movl 0(%rsi),%r10d // (NEXT STEP) X[0] movl $0xffffffff,%r11d - xorl %edx,%r11d - leal -198630844(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 28(%rsi),%r10d + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + leal -198630844(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 28(%rsi),%r10d // (NEXT STEP) X[7] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal 1126891415(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 56(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal 1126891415(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 56(%rsi),%r10d // (NEXT STEP) X[14] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal -1416354905(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 20(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal -1416354905(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 20(%rsi),%r10d // (NEXT STEP) X[5] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal -57434055(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 48(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal -57434055(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 48(%rsi),%r10d // (NEXT STEP) X[12] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - leal 1700485571(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 12(%rsi),%r10d + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x + leal 1700485571(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 12(%rsi),%r10d // (NEXT STEP) X[3] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal -1894986606(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 40(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal -1894986606(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 40(%rsi),%r10d // (NEXT STEP) X[10] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal -1051523(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 4(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal -1051523(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 4(%rsi),%r10d // (NEXT STEP) X[1] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal -2054922799(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 32(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal -2054922799(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 32(%rsi),%r10d // (NEXT STEP) X[8] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - leal 1873313359(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 60(%rsi),%r10d + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x + leal 1873313359(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 60(%rsi),%r10d // (NEXT STEP) X[15] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal -30611744(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 24(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal -30611744(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 24(%rsi),%r10d // (NEXT STEP) X[6] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal -1560198380(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 52(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal -1560198380(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 52(%rsi),%r10d // (NEXT STEP) X[13] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal 1309151649(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 16(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal 1309151649(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 16(%rsi),%r10d // (NEXT STEP) X[4] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - leal -145523070(%rax,%r10,1),%eax - orl %ebx,%r11d - xorl %ecx,%r11d - addl %r11d,%eax - movl 44(%rsi),%r10d + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x + leal -145523070(%rax,%r10,1),%eax // Const + dst + ... + orl %ebx,%r11d // x | ... + xorl %ecx,%r11d // y ^ ... + addl %r11d,%eax // dst += ... + movl 44(%rsi),%r10d // (NEXT STEP) X[11] movl $0xffffffff,%r11d - roll $6,%eax - xorl %ecx,%r11d - addl %ebx,%eax - leal -1120210379(%rdx,%r10,1),%edx - orl %eax,%r11d - xorl %ebx,%r11d - addl %r11d,%edx - movl 8(%rsi),%r10d + roll $6,%eax // dst <<< s + xorl %ecx,%r11d // (NEXT STEP) not z' = not %ecx + addl %ebx,%eax // dst += x + leal -1120210379(%rdx,%r10,1),%edx // Const + dst + ... + orl %eax,%r11d // x | ... + xorl %ebx,%r11d // y ^ ... + addl %r11d,%edx // dst += ... + movl 8(%rsi),%r10d // (NEXT STEP) X[2] movl $0xffffffff,%r11d - roll $10,%edx - xorl %ebx,%r11d - addl %eax,%edx - leal 718787259(%rcx,%r10,1),%ecx - orl %edx,%r11d - xorl %eax,%r11d - addl %r11d,%ecx - movl 36(%rsi),%r10d + roll $10,%edx // dst <<< s + xorl %ebx,%r11d // (NEXT STEP) not z' = not %ebx + addl %eax,%edx // dst += x + leal 718787259(%rcx,%r10,1),%ecx // Const + dst + ... + orl %edx,%r11d // x | ... + xorl %eax,%r11d // y ^ ... + addl %r11d,%ecx // dst += ... + movl 36(%rsi),%r10d // (NEXT STEP) X[9] movl $0xffffffff,%r11d - roll $15,%ecx - xorl %eax,%r11d - addl %edx,%ecx - leal -343485551(%rbx,%r10,1),%ebx - orl %ecx,%r11d - xorl %edx,%r11d - addl %r11d,%ebx - movl 0(%rsi),%r10d + roll $15,%ecx // dst <<< s + xorl %eax,%r11d // (NEXT STEP) not z' = not %eax + addl %edx,%ecx // dst += x + leal -343485551(%rbx,%r10,1),%ebx // Const + dst + ... + orl %ecx,%r11d // x | ... + xorl %edx,%r11d // y ^ ... + addl %r11d,%ebx // dst += ... + movl 0(%rsi),%r10d // (NEXT STEP) X[0] movl $0xffffffff,%r11d - roll $21,%ebx - xorl %edx,%r11d - addl %ecx,%ebx - + roll $21,%ebx // dst <<< s + xorl %edx,%r11d // (NEXT STEP) not z' = not %edx + addl %ecx,%ebx // dst += x +// add old values of A, B, C, D addl %r8d,%eax addl %r9d,%ebx addl %r14d,%ecx addl %r15d,%edx - - addq $64,%rsi - cmpq %rdi,%rsi - jb .Lloop - +// loop control + addq $64,%rsi // ptr += 64 + cmpq %rdi,%rsi // cmp end with ptr + jb .Lloop // jmp if ptr < end +// END of loop over 16-word blocks .Lend: - movl %eax,0(%rbp) - movl %ebx,4(%rbp) - movl %ecx,8(%rbp) - movl %edx,12(%rbp) + movl %eax,0(%rbp) // ctx->A = A + movl %ebx,4(%rbp) // ctx->B = B + movl %ecx,8(%rbp) // ctx->C = C + movl %edx,12(%rbp) // ctx->D = D movq (%rsp),%r15 .cfi_restore r15
diff --git a/gen/crypto/md5-x86_64-win.asm b/gen/crypto/md5-x86_64-win.asm index 5cfac97..8514872 100644 --- a/gen/crypto/md5-x86_64-win.asm +++ b/gen/crypto/md5-x86_64-win.asm
@@ -41,651 +41,651 @@ $L$prologue: +; rdi = arg #1 (ctx, MD5_CTX pointer) +; rsi = arg #2 (ptr, data pointer) +; rdx = arg #3 (nbr, number of 16-word blocks to process) + mov rbp,rdi ; rbp = ctx + shl rdx,6 ; rdx = nbr in bytes + lea rdi,[rdx*1+rsi] ; rdi = end + mov eax,DWORD[rbp] ; eax = ctx->A + mov ebx,DWORD[4+rbp] ; ebx = ctx->B + mov ecx,DWORD[8+rbp] ; ecx = ctx->C + mov edx,DWORD[12+rbp] ; edx = ctx->D +; end is 'rdi' +; ptr is 'rsi' +; A is 'eax' +; B is 'ebx' +; C is 'ecx' +; D is 'edx' + cmp rsi,rdi ; cmp end with ptr + je NEAR $L$end ; jmp if ptr == end - - mov rbp,rdi - shl rdx,6 - lea rdi,[rdx*1+rsi] - mov eax,DWORD[rbp] - mov ebx,DWORD[4+rbp] - mov ecx,DWORD[8+rbp] - mov edx,DWORD[12+rbp] - - - - - - - - cmp rsi,rdi - je NEAR $L$end - - -$L$loop: +; BEGIN of loop over 16-word blocks +$L$loop: ; save old values of A, B, C, D mov r8d,eax mov r9d,ebx mov r14d,ecx mov r15d,edx - mov r10d,DWORD[rsi] - mov r11d,edx - xor r11d,ecx - lea eax,[((-680876936))+r10*1+rax] - and r11d,ebx - xor r11d,edx - mov r10d,DWORD[4+rsi] - add eax,r11d - rol eax,7 - mov r11d,ecx - add eax,ebx - xor r11d,ebx - lea edx,[((-389564586))+r10*1+rdx] - and r11d,eax - xor r11d,ecx - mov r10d,DWORD[8+rsi] - add edx,r11d - rol edx,12 - mov r11d,ebx - add edx,eax - xor r11d,eax - lea ecx,[606105819+r10*1+rcx] - and r11d,edx - xor r11d,ebx - mov r10d,DWORD[12+rsi] - add ecx,r11d - rol ecx,17 - mov r11d,eax - add ecx,edx - xor r11d,edx - lea ebx,[((-1044525330))+r10*1+rbx] - and r11d,ecx - xor r11d,eax - mov r10d,DWORD[16+rsi] - add ebx,r11d - rol ebx,22 - mov r11d,edx - add ebx,ecx - xor r11d,ecx - lea eax,[((-176418897))+r10*1+rax] - and r11d,ebx - xor r11d,edx - mov r10d,DWORD[20+rsi] - add eax,r11d - rol eax,7 - mov r11d,ecx - add eax,ebx - xor r11d,ebx - lea edx,[1200080426+r10*1+rdx] - and r11d,eax - xor r11d,ecx - mov r10d,DWORD[24+rsi] - add edx,r11d - rol edx,12 - mov r11d,ebx - add edx,eax - xor r11d,eax - lea ecx,[((-1473231341))+r10*1+rcx] - and r11d,edx - xor r11d,ebx - mov r10d,DWORD[28+rsi] - add ecx,r11d - rol ecx,17 - mov r11d,eax - add ecx,edx - xor r11d,edx - lea ebx,[((-45705983))+r10*1+rbx] - and r11d,ecx - xor r11d,eax - mov r10d,DWORD[32+rsi] - add ebx,r11d - rol ebx,22 - mov r11d,edx - add ebx,ecx - xor r11d,ecx - lea eax,[1770035416+r10*1+rax] - and r11d,ebx - xor r11d,edx - mov r10d,DWORD[36+rsi] - add eax,r11d - rol eax,7 - mov r11d,ecx - add eax,ebx - xor r11d,ebx - lea edx,[((-1958414417))+r10*1+rdx] - and r11d,eax - xor r11d,ecx - mov r10d,DWORD[40+rsi] - add edx,r11d - rol edx,12 - mov r11d,ebx - add edx,eax - xor r11d,eax - lea ecx,[((-42063))+r10*1+rcx] - and r11d,edx - xor r11d,ebx - mov r10d,DWORD[44+rsi] - add ecx,r11d - rol ecx,17 - mov r11d,eax - add ecx,edx - xor r11d,edx - lea ebx,[((-1990404162))+r10*1+rbx] - and r11d,ecx - xor r11d,eax - mov r10d,DWORD[48+rsi] - add ebx,r11d - rol ebx,22 - mov r11d,edx - add ebx,ecx - xor r11d,ecx - lea eax,[1804603682+r10*1+rax] - and r11d,ebx - xor r11d,edx - mov r10d,DWORD[52+rsi] - add eax,r11d - rol eax,7 - mov r11d,ecx - add eax,ebx - xor r11d,ebx - lea edx,[((-40341101))+r10*1+rdx] - and r11d,eax - xor r11d,ecx - mov r10d,DWORD[56+rsi] - add edx,r11d - rol edx,12 - mov r11d,ebx - add edx,eax - xor r11d,eax - lea ecx,[((-1502002290))+r10*1+rcx] - and r11d,edx - xor r11d,ebx - mov r10d,DWORD[60+rsi] - add ecx,r11d - rol ecx,17 - mov r11d,eax - add ecx,edx - xor r11d,edx - lea ebx,[1236535329+r10*1+rbx] - and r11d,ecx - xor r11d,eax - mov r10d,DWORD[rsi] - add ebx,r11d - rol ebx,22 - mov r11d,edx - add ebx,ecx - mov r10d,DWORD[4+rsi] - mov r11d,edx - mov r12d,edx - not r11d - lea eax,[((-165796510))+r10*1+rax] - and r12d,ebx - and r11d,ecx - mov r10d,DWORD[24+rsi] - or r12d,r11d - mov r11d,ecx - add eax,r12d - mov r12d,ecx - rol eax,5 - add eax,ebx - not r11d - lea edx,[((-1069501632))+r10*1+rdx] - and r12d,eax - and r11d,ebx - mov r10d,DWORD[44+rsi] - or r12d,r11d - mov r11d,ebx - add edx,r12d - mov r12d,ebx - rol edx,9 - add edx,eax - not r11d - lea ecx,[643717713+r10*1+rcx] - and r12d,edx - and r11d,eax - mov r10d,DWORD[rsi] - or r12d,r11d - mov r11d,eax - add ecx,r12d - mov r12d,eax - rol ecx,14 - add ecx,edx - not r11d - lea ebx,[((-373897302))+r10*1+rbx] - and r12d,ecx - and r11d,edx - mov r10d,DWORD[20+rsi] - or r12d,r11d - mov r11d,edx - add ebx,r12d - mov r12d,edx - rol ebx,20 - add ebx,ecx - not r11d - lea eax,[((-701558691))+r10*1+rax] - and r12d,ebx - and r11d,ecx - mov r10d,DWORD[40+rsi] - or r12d,r11d - mov r11d,ecx - add eax,r12d - mov r12d,ecx - rol eax,5 - add eax,ebx - not r11d - lea edx,[38016083+r10*1+rdx] - and r12d,eax - and r11d,ebx - mov r10d,DWORD[60+rsi] - or r12d,r11d - mov r11d,ebx - add edx,r12d - mov r12d,ebx - rol edx,9 - add edx,eax - not r11d - lea ecx,[((-660478335))+r10*1+rcx] - and r12d,edx - and r11d,eax - mov r10d,DWORD[16+rsi] - or r12d,r11d - mov r11d,eax - add ecx,r12d - mov r12d,eax - rol ecx,14 - add ecx,edx - not r11d - lea ebx,[((-405537848))+r10*1+rbx] - and r12d,ecx - and r11d,edx - mov r10d,DWORD[36+rsi] - or r12d,r11d - mov r11d,edx - add ebx,r12d - mov r12d,edx - rol ebx,20 - add ebx,ecx - not r11d - lea eax,[568446438+r10*1+rax] - and r12d,ebx - and r11d,ecx - mov r10d,DWORD[56+rsi] - or r12d,r11d - mov r11d,ecx - add eax,r12d - mov r12d,ecx - rol eax,5 - add eax,ebx - not r11d - lea edx,[((-1019803690))+r10*1+rdx] - and r12d,eax - and r11d,ebx - mov r10d,DWORD[12+rsi] - or r12d,r11d - mov r11d,ebx - add edx,r12d - mov r12d,ebx - rol edx,9 - add edx,eax - not r11d - lea ecx,[((-187363961))+r10*1+rcx] - and r12d,edx - and r11d,eax - mov r10d,DWORD[32+rsi] - or r12d,r11d - mov r11d,eax - add ecx,r12d - mov r12d,eax - rol ecx,14 - add ecx,edx - not r11d - lea ebx,[1163531501+r10*1+rbx] - and r12d,ecx - and r11d,edx - mov r10d,DWORD[52+rsi] - or r12d,r11d - mov r11d,edx - add ebx,r12d - mov r12d,edx - rol ebx,20 - add ebx,ecx - not r11d - lea eax,[((-1444681467))+r10*1+rax] - and r12d,ebx - and r11d,ecx - mov r10d,DWORD[8+rsi] - or r12d,r11d - mov r11d,ecx - add eax,r12d - mov r12d,ecx - rol eax,5 - add eax,ebx - not r11d - lea edx,[((-51403784))+r10*1+rdx] - and r12d,eax - and r11d,ebx - mov r10d,DWORD[28+rsi] - or r12d,r11d - mov r11d,ebx - add edx,r12d - mov r12d,ebx - rol edx,9 - add edx,eax - not r11d - lea ecx,[1735328473+r10*1+rcx] - and r12d,edx - and r11d,eax - mov r10d,DWORD[48+rsi] - or r12d,r11d - mov r11d,eax - add ecx,r12d - mov r12d,eax - rol ecx,14 - add ecx,edx - not r11d - lea ebx,[((-1926607734))+r10*1+rbx] - and r12d,ecx - and r11d,edx - mov r10d,DWORD[rsi] - or r12d,r11d - mov r11d,edx - add ebx,r12d - mov r12d,edx - rol ebx,20 - add ebx,ecx - mov r10d,DWORD[20+rsi] - mov r11d,ecx - lea eax,[((-378558))+r10*1+rax] - mov r10d,DWORD[32+rsi] - xor r11d,edx - xor r11d,ebx - add eax,r11d - rol eax,4 - mov r11d,ebx - add eax,ebx - lea edx,[((-2022574463))+r10*1+rdx] - mov r10d,DWORD[44+rsi] - xor r11d,ecx - xor r11d,eax - add edx,r11d - rol edx,11 - mov r11d,eax - add edx,eax - lea ecx,[1839030562+r10*1+rcx] - mov r10d,DWORD[56+rsi] - xor r11d,ebx - xor r11d,edx - add ecx,r11d - rol ecx,16 - mov r11d,edx - add ecx,edx - lea ebx,[((-35309556))+r10*1+rbx] - mov r10d,DWORD[4+rsi] - xor r11d,eax - xor r11d,ecx - add ebx,r11d - rol ebx,23 - mov r11d,ecx - add ebx,ecx - lea eax,[((-1530992060))+r10*1+rax] - mov r10d,DWORD[16+rsi] - xor r11d,edx - xor r11d,ebx - add eax,r11d - rol eax,4 - mov r11d,ebx - add eax,ebx - lea edx,[1272893353+r10*1+rdx] - mov r10d,DWORD[28+rsi] - xor r11d,ecx - xor r11d,eax - add edx,r11d - rol edx,11 - mov r11d,eax - add edx,eax - lea ecx,[((-155497632))+r10*1+rcx] - mov r10d,DWORD[40+rsi] - xor r11d,ebx - xor r11d,edx - add ecx,r11d - rol ecx,16 - mov r11d,edx - add ecx,edx - lea ebx,[((-1094730640))+r10*1+rbx] - mov r10d,DWORD[52+rsi] - xor r11d,eax - xor r11d,ecx - add ebx,r11d - rol ebx,23 - mov r11d,ecx - add ebx,ecx - lea eax,[681279174+r10*1+rax] - mov r10d,DWORD[rsi] - xor r11d,edx - xor r11d,ebx - add eax,r11d - rol eax,4 - mov r11d,ebx - add eax,ebx - lea edx,[((-358537222))+r10*1+rdx] - mov r10d,DWORD[12+rsi] - xor r11d,ecx - xor r11d,eax - add edx,r11d - rol edx,11 - mov r11d,eax - add edx,eax - lea ecx,[((-722521979))+r10*1+rcx] - mov r10d,DWORD[24+rsi] - xor r11d,ebx - xor r11d,edx - add ecx,r11d - rol ecx,16 - mov r11d,edx - add ecx,edx - lea ebx,[76029189+r10*1+rbx] - mov r10d,DWORD[36+rsi] - xor r11d,eax - xor r11d,ecx - add ebx,r11d - rol ebx,23 - mov r11d,ecx - add ebx,ecx - lea eax,[((-640364487))+r10*1+rax] - mov r10d,DWORD[48+rsi] - xor r11d,edx - xor r11d,ebx - add eax,r11d - rol eax,4 - mov r11d,ebx - add eax,ebx - lea edx,[((-421815835))+r10*1+rdx] - mov r10d,DWORD[60+rsi] - xor r11d,ecx - xor r11d,eax - add edx,r11d - rol edx,11 - mov r11d,eax - add edx,eax - lea ecx,[530742520+r10*1+rcx] - mov r10d,DWORD[8+rsi] - xor r11d,ebx - xor r11d,edx - add ecx,r11d - rol ecx,16 - mov r11d,edx - add ecx,edx - lea ebx,[((-995338651))+r10*1+rbx] - mov r10d,DWORD[rsi] - xor r11d,eax - xor r11d,ecx - add ebx,r11d - rol ebx,23 - mov r11d,ecx - add ebx,ecx - mov r10d,DWORD[rsi] + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] + mov r11d,edx ; (NEXT STEP) z' = %edx + xor r11d,ecx ; y ^ ... + lea eax,[((-680876936))+r10*1+rax] ; Const + dst + ... + and r11d,ebx ; x & ... + xor r11d,edx ; z ^ ... + mov r10d,DWORD[4+rsi] ; (NEXT STEP) X[1] + add eax,r11d ; dst += ... + rol eax,7 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,ebx ; dst += x + xor r11d,ebx ; y ^ ... + lea edx,[((-389564586))+r10*1+rdx] ; Const + dst + ... + and r11d,eax ; x & ... + xor r11d,ecx ; z ^ ... + mov r10d,DWORD[8+rsi] ; (NEXT STEP) X[2] + add edx,r11d ; dst += ... + rol edx,12 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,eax ; dst += x + xor r11d,eax ; y ^ ... + lea ecx,[606105819+r10*1+rcx] ; Const + dst + ... + and r11d,edx ; x & ... + xor r11d,ebx ; z ^ ... + mov r10d,DWORD[12+rsi] ; (NEXT STEP) X[3] + add ecx,r11d ; dst += ... + rol ecx,17 ; dst <<< s + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,edx ; dst += x + xor r11d,edx ; y ^ ... + lea ebx,[((-1044525330))+r10*1+rbx] ; Const + dst + ... + and r11d,ecx ; x & ... + xor r11d,eax ; z ^ ... + mov r10d,DWORD[16+rsi] ; (NEXT STEP) X[4] + add ebx,r11d ; dst += ... + rol ebx,22 ; dst <<< s + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,ecx ; dst += x + xor r11d,ecx ; y ^ ... + lea eax,[((-176418897))+r10*1+rax] ; Const + dst + ... + and r11d,ebx ; x & ... + xor r11d,edx ; z ^ ... + mov r10d,DWORD[20+rsi] ; (NEXT STEP) X[5] + add eax,r11d ; dst += ... + rol eax,7 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,ebx ; dst += x + xor r11d,ebx ; y ^ ... + lea edx,[1200080426+r10*1+rdx] ; Const + dst + ... + and r11d,eax ; x & ... + xor r11d,ecx ; z ^ ... + mov r10d,DWORD[24+rsi] ; (NEXT STEP) X[6] + add edx,r11d ; dst += ... + rol edx,12 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,eax ; dst += x + xor r11d,eax ; y ^ ... + lea ecx,[((-1473231341))+r10*1+rcx] ; Const + dst + ... + and r11d,edx ; x & ... + xor r11d,ebx ; z ^ ... + mov r10d,DWORD[28+rsi] ; (NEXT STEP) X[7] + add ecx,r11d ; dst += ... + rol ecx,17 ; dst <<< s + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,edx ; dst += x + xor r11d,edx ; y ^ ... + lea ebx,[((-45705983))+r10*1+rbx] ; Const + dst + ... + and r11d,ecx ; x & ... + xor r11d,eax ; z ^ ... + mov r10d,DWORD[32+rsi] ; (NEXT STEP) X[8] + add ebx,r11d ; dst += ... + rol ebx,22 ; dst <<< s + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,ecx ; dst += x + xor r11d,ecx ; y ^ ... + lea eax,[1770035416+r10*1+rax] ; Const + dst + ... + and r11d,ebx ; x & ... + xor r11d,edx ; z ^ ... + mov r10d,DWORD[36+rsi] ; (NEXT STEP) X[9] + add eax,r11d ; dst += ... + rol eax,7 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,ebx ; dst += x + xor r11d,ebx ; y ^ ... + lea edx,[((-1958414417))+r10*1+rdx] ; Const + dst + ... + and r11d,eax ; x & ... + xor r11d,ecx ; z ^ ... + mov r10d,DWORD[40+rsi] ; (NEXT STEP) X[10] + add edx,r11d ; dst += ... + rol edx,12 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,eax ; dst += x + xor r11d,eax ; y ^ ... + lea ecx,[((-42063))+r10*1+rcx] ; Const + dst + ... + and r11d,edx ; x & ... + xor r11d,ebx ; z ^ ... + mov r10d,DWORD[44+rsi] ; (NEXT STEP) X[11] + add ecx,r11d ; dst += ... + rol ecx,17 ; dst <<< s + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,edx ; dst += x + xor r11d,edx ; y ^ ... + lea ebx,[((-1990404162))+r10*1+rbx] ; Const + dst + ... + and r11d,ecx ; x & ... + xor r11d,eax ; z ^ ... + mov r10d,DWORD[48+rsi] ; (NEXT STEP) X[12] + add ebx,r11d ; dst += ... + rol ebx,22 ; dst <<< s + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,ecx ; dst += x + xor r11d,ecx ; y ^ ... + lea eax,[1804603682+r10*1+rax] ; Const + dst + ... + and r11d,ebx ; x & ... + xor r11d,edx ; z ^ ... + mov r10d,DWORD[52+rsi] ; (NEXT STEP) X[13] + add eax,r11d ; dst += ... + rol eax,7 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,ebx ; dst += x + xor r11d,ebx ; y ^ ... + lea edx,[((-40341101))+r10*1+rdx] ; Const + dst + ... + and r11d,eax ; x & ... + xor r11d,ecx ; z ^ ... + mov r10d,DWORD[56+rsi] ; (NEXT STEP) X[14] + add edx,r11d ; dst += ... + rol edx,12 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,eax ; dst += x + xor r11d,eax ; y ^ ... + lea ecx,[((-1502002290))+r10*1+rcx] ; Const + dst + ... + and r11d,edx ; x & ... + xor r11d,ebx ; z ^ ... + mov r10d,DWORD[60+rsi] ; (NEXT STEP) X[15] + add ecx,r11d ; dst += ... + rol ecx,17 ; dst <<< s + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,edx ; dst += x + xor r11d,edx ; y ^ ... + lea ebx,[1236535329+r10*1+rbx] ; Const + dst + ... + and r11d,ecx ; x & ... + xor r11d,eax ; z ^ ... + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] + add ebx,r11d ; dst += ... + rol ebx,22 ; dst <<< s + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,ecx ; dst += x + mov r10d,DWORD[4+rsi] ; (NEXT STEP) X[1] + mov r11d,edx ; (NEXT STEP) z' = %edx + mov r12d,edx ; (NEXT STEP) z' = %edx + not r11d ; not z + lea eax,[((-165796510))+r10*1+rax] ; Const + dst + ... + and r12d,ebx ; x & z + and r11d,ecx ; y & (not z) + mov r10d,DWORD[24+rsi] ; (NEXT STEP) X[6] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,r12d ; dst += ... + mov r12d,ecx ; (NEXT STEP) z' = %ecx + rol eax,5 ; dst <<< s + add eax,ebx ; dst += x + not r11d ; not z + lea edx,[((-1069501632))+r10*1+rdx] ; Const + dst + ... + and r12d,eax ; x & z + and r11d,ebx ; y & (not z) + mov r10d,DWORD[44+rsi] ; (NEXT STEP) X[11] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,r12d ; dst += ... + mov r12d,ebx ; (NEXT STEP) z' = %ebx + rol edx,9 ; dst <<< s + add edx,eax ; dst += x + not r11d ; not z + lea ecx,[643717713+r10*1+rcx] ; Const + dst + ... + and r12d,edx ; x & z + and r11d,eax ; y & (not z) + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,r12d ; dst += ... + mov r12d,eax ; (NEXT STEP) z' = %eax + rol ecx,14 ; dst <<< s + add ecx,edx ; dst += x + not r11d ; not z + lea ebx,[((-373897302))+r10*1+rbx] ; Const + dst + ... + and r12d,ecx ; x & z + and r11d,edx ; y & (not z) + mov r10d,DWORD[20+rsi] ; (NEXT STEP) X[5] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,r12d ; dst += ... + mov r12d,edx ; (NEXT STEP) z' = %edx + rol ebx,20 ; dst <<< s + add ebx,ecx ; dst += x + not r11d ; not z + lea eax,[((-701558691))+r10*1+rax] ; Const + dst + ... + and r12d,ebx ; x & z + and r11d,ecx ; y & (not z) + mov r10d,DWORD[40+rsi] ; (NEXT STEP) X[10] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,r12d ; dst += ... + mov r12d,ecx ; (NEXT STEP) z' = %ecx + rol eax,5 ; dst <<< s + add eax,ebx ; dst += x + not r11d ; not z + lea edx,[38016083+r10*1+rdx] ; Const + dst + ... + and r12d,eax ; x & z + and r11d,ebx ; y & (not z) + mov r10d,DWORD[60+rsi] ; (NEXT STEP) X[15] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,r12d ; dst += ... + mov r12d,ebx ; (NEXT STEP) z' = %ebx + rol edx,9 ; dst <<< s + add edx,eax ; dst += x + not r11d ; not z + lea ecx,[((-660478335))+r10*1+rcx] ; Const + dst + ... + and r12d,edx ; x & z + and r11d,eax ; y & (not z) + mov r10d,DWORD[16+rsi] ; (NEXT STEP) X[4] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,r12d ; dst += ... + mov r12d,eax ; (NEXT STEP) z' = %eax + rol ecx,14 ; dst <<< s + add ecx,edx ; dst += x + not r11d ; not z + lea ebx,[((-405537848))+r10*1+rbx] ; Const + dst + ... + and r12d,ecx ; x & z + and r11d,edx ; y & (not z) + mov r10d,DWORD[36+rsi] ; (NEXT STEP) X[9] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,r12d ; dst += ... + mov r12d,edx ; (NEXT STEP) z' = %edx + rol ebx,20 ; dst <<< s + add ebx,ecx ; dst += x + not r11d ; not z + lea eax,[568446438+r10*1+rax] ; Const + dst + ... + and r12d,ebx ; x & z + and r11d,ecx ; y & (not z) + mov r10d,DWORD[56+rsi] ; (NEXT STEP) X[14] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,r12d ; dst += ... + mov r12d,ecx ; (NEXT STEP) z' = %ecx + rol eax,5 ; dst <<< s + add eax,ebx ; dst += x + not r11d ; not z + lea edx,[((-1019803690))+r10*1+rdx] ; Const + dst + ... + and r12d,eax ; x & z + and r11d,ebx ; y & (not z) + mov r10d,DWORD[12+rsi] ; (NEXT STEP) X[3] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,r12d ; dst += ... + mov r12d,ebx ; (NEXT STEP) z' = %ebx + rol edx,9 ; dst <<< s + add edx,eax ; dst += x + not r11d ; not z + lea ecx,[((-187363961))+r10*1+rcx] ; Const + dst + ... + and r12d,edx ; x & z + and r11d,eax ; y & (not z) + mov r10d,DWORD[32+rsi] ; (NEXT STEP) X[8] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,r12d ; dst += ... + mov r12d,eax ; (NEXT STEP) z' = %eax + rol ecx,14 ; dst <<< s + add ecx,edx ; dst += x + not r11d ; not z + lea ebx,[1163531501+r10*1+rbx] ; Const + dst + ... + and r12d,ecx ; x & z + and r11d,edx ; y & (not z) + mov r10d,DWORD[52+rsi] ; (NEXT STEP) X[13] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,r12d ; dst += ... + mov r12d,edx ; (NEXT STEP) z' = %edx + rol ebx,20 ; dst <<< s + add ebx,ecx ; dst += x + not r11d ; not z + lea eax,[((-1444681467))+r10*1+rax] ; Const + dst + ... + and r12d,ebx ; x & z + and r11d,ecx ; y & (not z) + mov r10d,DWORD[8+rsi] ; (NEXT STEP) X[2] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ecx ; (NEXT STEP) z' = %ecx + add eax,r12d ; dst += ... + mov r12d,ecx ; (NEXT STEP) z' = %ecx + rol eax,5 ; dst <<< s + add eax,ebx ; dst += x + not r11d ; not z + lea edx,[((-51403784))+r10*1+rdx] ; Const + dst + ... + and r12d,eax ; x & z + and r11d,ebx ; y & (not z) + mov r10d,DWORD[28+rsi] ; (NEXT STEP) X[7] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,ebx ; (NEXT STEP) z' = %ebx + add edx,r12d ; dst += ... + mov r12d,ebx ; (NEXT STEP) z' = %ebx + rol edx,9 ; dst <<< s + add edx,eax ; dst += x + not r11d ; not z + lea ecx,[1735328473+r10*1+rcx] ; Const + dst + ... + and r12d,edx ; x & z + and r11d,eax ; y & (not z) + mov r10d,DWORD[48+rsi] ; (NEXT STEP) X[12] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,eax ; (NEXT STEP) z' = %eax + add ecx,r12d ; dst += ... + mov r12d,eax ; (NEXT STEP) z' = %eax + rol ecx,14 ; dst <<< s + add ecx,edx ; dst += x + not r11d ; not z + lea ebx,[((-1926607734))+r10*1+rbx] ; Const + dst + ... + and r12d,ecx ; x & z + and r11d,edx ; y & (not z) + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] + or r12d,r11d ; (y & (not z)) | (x & z) + mov r11d,edx ; (NEXT STEP) z' = %edx + add ebx,r12d ; dst += ... + mov r12d,edx ; (NEXT STEP) z' = %edx + rol ebx,20 ; dst <<< s + add ebx,ecx ; dst += x + mov r10d,DWORD[20+rsi] ; (NEXT STEP) X[5] + mov r11d,ecx ; (NEXT STEP) y' = %ecx + lea eax,[((-378558))+r10*1+rax] ; Const + dst + ... + mov r10d,DWORD[32+rsi] ; (NEXT STEP) X[8] + xor r11d,edx ; z ^ ... + xor r11d,ebx ; x ^ ... + add eax,r11d ; dst += ... + rol eax,4 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) y' = %ebx + add eax,ebx ; dst += x + lea edx,[((-2022574463))+r10*1+rdx] ; Const + dst + ... + mov r10d,DWORD[44+rsi] ; (NEXT STEP) X[11] + xor r11d,ecx ; z ^ ... + xor r11d,eax ; x ^ ... + add edx,r11d ; dst += ... + rol edx,11 ; dst <<< s + mov r11d,eax ; (NEXT STEP) y' = %eax + add edx,eax ; dst += x + lea ecx,[1839030562+r10*1+rcx] ; Const + dst + ... + mov r10d,DWORD[56+rsi] ; (NEXT STEP) X[14] + xor r11d,ebx ; z ^ ... + xor r11d,edx ; x ^ ... + add ecx,r11d ; dst += ... + rol ecx,16 ; dst <<< s + mov r11d,edx ; (NEXT STEP) y' = %edx + add ecx,edx ; dst += x + lea ebx,[((-35309556))+r10*1+rbx] ; Const + dst + ... + mov r10d,DWORD[4+rsi] ; (NEXT STEP) X[1] + xor r11d,eax ; z ^ ... + xor r11d,ecx ; x ^ ... + add ebx,r11d ; dst += ... + rol ebx,23 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) y' = %ecx + add ebx,ecx ; dst += x + lea eax,[((-1530992060))+r10*1+rax] ; Const + dst + ... + mov r10d,DWORD[16+rsi] ; (NEXT STEP) X[4] + xor r11d,edx ; z ^ ... + xor r11d,ebx ; x ^ ... + add eax,r11d ; dst += ... + rol eax,4 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) y' = %ebx + add eax,ebx ; dst += x + lea edx,[1272893353+r10*1+rdx] ; Const + dst + ... + mov r10d,DWORD[28+rsi] ; (NEXT STEP) X[7] + xor r11d,ecx ; z ^ ... + xor r11d,eax ; x ^ ... + add edx,r11d ; dst += ... + rol edx,11 ; dst <<< s + mov r11d,eax ; (NEXT STEP) y' = %eax + add edx,eax ; dst += x + lea ecx,[((-155497632))+r10*1+rcx] ; Const + dst + ... + mov r10d,DWORD[40+rsi] ; (NEXT STEP) X[10] + xor r11d,ebx ; z ^ ... + xor r11d,edx ; x ^ ... + add ecx,r11d ; dst += ... + rol ecx,16 ; dst <<< s + mov r11d,edx ; (NEXT STEP) y' = %edx + add ecx,edx ; dst += x + lea ebx,[((-1094730640))+r10*1+rbx] ; Const + dst + ... + mov r10d,DWORD[52+rsi] ; (NEXT STEP) X[13] + xor r11d,eax ; z ^ ... + xor r11d,ecx ; x ^ ... + add ebx,r11d ; dst += ... + rol ebx,23 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) y' = %ecx + add ebx,ecx ; dst += x + lea eax,[681279174+r10*1+rax] ; Const + dst + ... + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] + xor r11d,edx ; z ^ ... + xor r11d,ebx ; x ^ ... + add eax,r11d ; dst += ... + rol eax,4 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) y' = %ebx + add eax,ebx ; dst += x + lea edx,[((-358537222))+r10*1+rdx] ; Const + dst + ... + mov r10d,DWORD[12+rsi] ; (NEXT STEP) X[3] + xor r11d,ecx ; z ^ ... + xor r11d,eax ; x ^ ... + add edx,r11d ; dst += ... + rol edx,11 ; dst <<< s + mov r11d,eax ; (NEXT STEP) y' = %eax + add edx,eax ; dst += x + lea ecx,[((-722521979))+r10*1+rcx] ; Const + dst + ... + mov r10d,DWORD[24+rsi] ; (NEXT STEP) X[6] + xor r11d,ebx ; z ^ ... + xor r11d,edx ; x ^ ... + add ecx,r11d ; dst += ... + rol ecx,16 ; dst <<< s + mov r11d,edx ; (NEXT STEP) y' = %edx + add ecx,edx ; dst += x + lea ebx,[76029189+r10*1+rbx] ; Const + dst + ... + mov r10d,DWORD[36+rsi] ; (NEXT STEP) X[9] + xor r11d,eax ; z ^ ... + xor r11d,ecx ; x ^ ... + add ebx,r11d ; dst += ... + rol ebx,23 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) y' = %ecx + add ebx,ecx ; dst += x + lea eax,[((-640364487))+r10*1+rax] ; Const + dst + ... + mov r10d,DWORD[48+rsi] ; (NEXT STEP) X[12] + xor r11d,edx ; z ^ ... + xor r11d,ebx ; x ^ ... + add eax,r11d ; dst += ... + rol eax,4 ; dst <<< s + mov r11d,ebx ; (NEXT STEP) y' = %ebx + add eax,ebx ; dst += x + lea edx,[((-421815835))+r10*1+rdx] ; Const + dst + ... + mov r10d,DWORD[60+rsi] ; (NEXT STEP) X[15] + xor r11d,ecx ; z ^ ... + xor r11d,eax ; x ^ ... + add edx,r11d ; dst += ... + rol edx,11 ; dst <<< s + mov r11d,eax ; (NEXT STEP) y' = %eax + add edx,eax ; dst += x + lea ecx,[530742520+r10*1+rcx] ; Const + dst + ... + mov r10d,DWORD[8+rsi] ; (NEXT STEP) X[2] + xor r11d,ebx ; z ^ ... + xor r11d,edx ; x ^ ... + add ecx,r11d ; dst += ... + rol ecx,16 ; dst <<< s + mov r11d,edx ; (NEXT STEP) y' = %edx + add ecx,edx ; dst += x + lea ebx,[((-995338651))+r10*1+rbx] ; Const + dst + ... + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] + xor r11d,eax ; z ^ ... + xor r11d,ecx ; x ^ ... + add ebx,r11d ; dst += ... + rol ebx,23 ; dst <<< s + mov r11d,ecx ; (NEXT STEP) y' = %ecx + add ebx,ecx ; dst += x + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] mov r11d,0xffffffff - xor r11d,edx - lea eax,[((-198630844))+r10*1+rax] - or r11d,ebx - xor r11d,ecx - add eax,r11d - mov r10d,DWORD[28+rsi] + xor r11d,edx ; (NEXT STEP) not z' = not %edx + lea eax,[((-198630844))+r10*1+rax] ; Const + dst + ... + or r11d,ebx ; x | ... + xor r11d,ecx ; y ^ ... + add eax,r11d ; dst += ... + mov r10d,DWORD[28+rsi] ; (NEXT STEP) X[7] mov r11d,0xffffffff - rol eax,6 - xor r11d,ecx - add eax,ebx - lea edx,[1126891415+r10*1+rdx] - or r11d,eax - xor r11d,ebx - add edx,r11d - mov r10d,DWORD[56+rsi] + rol eax,6 ; dst <<< s + xor r11d,ecx ; (NEXT STEP) not z' = not %ecx + add eax,ebx ; dst += x + lea edx,[1126891415+r10*1+rdx] ; Const + dst + ... + or r11d,eax ; x | ... + xor r11d,ebx ; y ^ ... + add edx,r11d ; dst += ... + mov r10d,DWORD[56+rsi] ; (NEXT STEP) X[14] mov r11d,0xffffffff - rol edx,10 - xor r11d,ebx - add edx,eax - lea ecx,[((-1416354905))+r10*1+rcx] - or r11d,edx - xor r11d,eax - add ecx,r11d - mov r10d,DWORD[20+rsi] + rol edx,10 ; dst <<< s + xor r11d,ebx ; (NEXT STEP) not z' = not %ebx + add edx,eax ; dst += x + lea ecx,[((-1416354905))+r10*1+rcx] ; Const + dst + ... + or r11d,edx ; x | ... + xor r11d,eax ; y ^ ... + add ecx,r11d ; dst += ... + mov r10d,DWORD[20+rsi] ; (NEXT STEP) X[5] mov r11d,0xffffffff - rol ecx,15 - xor r11d,eax - add ecx,edx - lea ebx,[((-57434055))+r10*1+rbx] - or r11d,ecx - xor r11d,edx - add ebx,r11d - mov r10d,DWORD[48+rsi] + rol ecx,15 ; dst <<< s + xor r11d,eax ; (NEXT STEP) not z' = not %eax + add ecx,edx ; dst += x + lea ebx,[((-57434055))+r10*1+rbx] ; Const + dst + ... + or r11d,ecx ; x | ... + xor r11d,edx ; y ^ ... + add ebx,r11d ; dst += ... + mov r10d,DWORD[48+rsi] ; (NEXT STEP) X[12] mov r11d,0xffffffff - rol ebx,21 - xor r11d,edx - add ebx,ecx - lea eax,[1700485571+r10*1+rax] - or r11d,ebx - xor r11d,ecx - add eax,r11d - mov r10d,DWORD[12+rsi] + rol ebx,21 ; dst <<< s + xor r11d,edx ; (NEXT STEP) not z' = not %edx + add ebx,ecx ; dst += x + lea eax,[1700485571+r10*1+rax] ; Const + dst + ... + or r11d,ebx ; x | ... + xor r11d,ecx ; y ^ ... + add eax,r11d ; dst += ... + mov r10d,DWORD[12+rsi] ; (NEXT STEP) X[3] mov r11d,0xffffffff - rol eax,6 - xor r11d,ecx - add eax,ebx - lea edx,[((-1894986606))+r10*1+rdx] - or r11d,eax - xor r11d,ebx - add edx,r11d - mov r10d,DWORD[40+rsi] + rol eax,6 ; dst <<< s + xor r11d,ecx ; (NEXT STEP) not z' = not %ecx + add eax,ebx ; dst += x + lea edx,[((-1894986606))+r10*1+rdx] ; Const + dst + ... + or r11d,eax ; x | ... + xor r11d,ebx ; y ^ ... + add edx,r11d ; dst += ... + mov r10d,DWORD[40+rsi] ; (NEXT STEP) X[10] mov r11d,0xffffffff - rol edx,10 - xor r11d,ebx - add edx,eax - lea ecx,[((-1051523))+r10*1+rcx] - or r11d,edx - xor r11d,eax - add ecx,r11d - mov r10d,DWORD[4+rsi] + rol edx,10 ; dst <<< s + xor r11d,ebx ; (NEXT STEP) not z' = not %ebx + add edx,eax ; dst += x + lea ecx,[((-1051523))+r10*1+rcx] ; Const + dst + ... + or r11d,edx ; x | ... + xor r11d,eax ; y ^ ... + add ecx,r11d ; dst += ... + mov r10d,DWORD[4+rsi] ; (NEXT STEP) X[1] mov r11d,0xffffffff - rol ecx,15 - xor r11d,eax - add ecx,edx - lea ebx,[((-2054922799))+r10*1+rbx] - or r11d,ecx - xor r11d,edx - add ebx,r11d - mov r10d,DWORD[32+rsi] + rol ecx,15 ; dst <<< s + xor r11d,eax ; (NEXT STEP) not z' = not %eax + add ecx,edx ; dst += x + lea ebx,[((-2054922799))+r10*1+rbx] ; Const + dst + ... + or r11d,ecx ; x | ... + xor r11d,edx ; y ^ ... + add ebx,r11d ; dst += ... + mov r10d,DWORD[32+rsi] ; (NEXT STEP) X[8] mov r11d,0xffffffff - rol ebx,21 - xor r11d,edx - add ebx,ecx - lea eax,[1873313359+r10*1+rax] - or r11d,ebx - xor r11d,ecx - add eax,r11d - mov r10d,DWORD[60+rsi] + rol ebx,21 ; dst <<< s + xor r11d,edx ; (NEXT STEP) not z' = not %edx + add ebx,ecx ; dst += x + lea eax,[1873313359+r10*1+rax] ; Const + dst + ... + or r11d,ebx ; x | ... + xor r11d,ecx ; y ^ ... + add eax,r11d ; dst += ... + mov r10d,DWORD[60+rsi] ; (NEXT STEP) X[15] mov r11d,0xffffffff - rol eax,6 - xor r11d,ecx - add eax,ebx - lea edx,[((-30611744))+r10*1+rdx] - or r11d,eax - xor r11d,ebx - add edx,r11d - mov r10d,DWORD[24+rsi] + rol eax,6 ; dst <<< s + xor r11d,ecx ; (NEXT STEP) not z' = not %ecx + add eax,ebx ; dst += x + lea edx,[((-30611744))+r10*1+rdx] ; Const + dst + ... + or r11d,eax ; x | ... + xor r11d,ebx ; y ^ ... + add edx,r11d ; dst += ... + mov r10d,DWORD[24+rsi] ; (NEXT STEP) X[6] mov r11d,0xffffffff - rol edx,10 - xor r11d,ebx - add edx,eax - lea ecx,[((-1560198380))+r10*1+rcx] - or r11d,edx - xor r11d,eax - add ecx,r11d - mov r10d,DWORD[52+rsi] + rol edx,10 ; dst <<< s + xor r11d,ebx ; (NEXT STEP) not z' = not %ebx + add edx,eax ; dst += x + lea ecx,[((-1560198380))+r10*1+rcx] ; Const + dst + ... + or r11d,edx ; x | ... + xor r11d,eax ; y ^ ... + add ecx,r11d ; dst += ... + mov r10d,DWORD[52+rsi] ; (NEXT STEP) X[13] mov r11d,0xffffffff - rol ecx,15 - xor r11d,eax - add ecx,edx - lea ebx,[1309151649+r10*1+rbx] - or r11d,ecx - xor r11d,edx - add ebx,r11d - mov r10d,DWORD[16+rsi] + rol ecx,15 ; dst <<< s + xor r11d,eax ; (NEXT STEP) not z' = not %eax + add ecx,edx ; dst += x + lea ebx,[1309151649+r10*1+rbx] ; Const + dst + ... + or r11d,ecx ; x | ... + xor r11d,edx ; y ^ ... + add ebx,r11d ; dst += ... + mov r10d,DWORD[16+rsi] ; (NEXT STEP) X[4] mov r11d,0xffffffff - rol ebx,21 - xor r11d,edx - add ebx,ecx - lea eax,[((-145523070))+r10*1+rax] - or r11d,ebx - xor r11d,ecx - add eax,r11d - mov r10d,DWORD[44+rsi] + rol ebx,21 ; dst <<< s + xor r11d,edx ; (NEXT STEP) not z' = not %edx + add ebx,ecx ; dst += x + lea eax,[((-145523070))+r10*1+rax] ; Const + dst + ... + or r11d,ebx ; x | ... + xor r11d,ecx ; y ^ ... + add eax,r11d ; dst += ... + mov r10d,DWORD[44+rsi] ; (NEXT STEP) X[11] mov r11d,0xffffffff - rol eax,6 - xor r11d,ecx - add eax,ebx - lea edx,[((-1120210379))+r10*1+rdx] - or r11d,eax - xor r11d,ebx - add edx,r11d - mov r10d,DWORD[8+rsi] + rol eax,6 ; dst <<< s + xor r11d,ecx ; (NEXT STEP) not z' = not %ecx + add eax,ebx ; dst += x + lea edx,[((-1120210379))+r10*1+rdx] ; Const + dst + ... + or r11d,eax ; x | ... + xor r11d,ebx ; y ^ ... + add edx,r11d ; dst += ... + mov r10d,DWORD[8+rsi] ; (NEXT STEP) X[2] mov r11d,0xffffffff - rol edx,10 - xor r11d,ebx - add edx,eax - lea ecx,[718787259+r10*1+rcx] - or r11d,edx - xor r11d,eax - add ecx,r11d - mov r10d,DWORD[36+rsi] + rol edx,10 ; dst <<< s + xor r11d,ebx ; (NEXT STEP) not z' = not %ebx + add edx,eax ; dst += x + lea ecx,[718787259+r10*1+rcx] ; Const + dst + ... + or r11d,edx ; x | ... + xor r11d,eax ; y ^ ... + add ecx,r11d ; dst += ... + mov r10d,DWORD[36+rsi] ; (NEXT STEP) X[9] mov r11d,0xffffffff - rol ecx,15 - xor r11d,eax - add ecx,edx - lea ebx,[((-343485551))+r10*1+rbx] - or r11d,ecx - xor r11d,edx - add ebx,r11d - mov r10d,DWORD[rsi] + rol ecx,15 ; dst <<< s + xor r11d,eax ; (NEXT STEP) not z' = not %eax + add ecx,edx ; dst += x + lea ebx,[((-343485551))+r10*1+rbx] ; Const + dst + ... + or r11d,ecx ; x | ... + xor r11d,edx ; y ^ ... + add ebx,r11d ; dst += ... + mov r10d,DWORD[rsi] ; (NEXT STEP) X[0] mov r11d,0xffffffff - rol ebx,21 - xor r11d,edx - add ebx,ecx - + rol ebx,21 ; dst <<< s + xor r11d,edx ; (NEXT STEP) not z' = not %edx + add ebx,ecx ; dst += x +; add old values of A, B, C, D add eax,r8d add ebx,r9d add ecx,r14d add edx,r15d - - add rsi,64 - cmp rsi,rdi - jb NEAR $L$loop - +; loop control + add rsi,64 ; ptr += 64 + cmp rsi,rdi ; cmp end with ptr + jb NEAR $L$loop ; jmp if ptr < end +; END of loop over 16-word blocks $L$end: - mov DWORD[rbp],eax - mov DWORD[4+rbp],ebx - mov DWORD[8+rbp],ecx - mov DWORD[12+rbp],edx + mov DWORD[rbp],eax ; ctx->A = A + mov DWORD[4+rbp],ebx ; ctx->B = B + mov DWORD[8+rbp],ecx ; ctx->C = C + mov DWORD[12+rbp],edx ; ctx->D = D mov r15,QWORD[rsp] @@ -720,17 +720,17 @@ pushfq sub rsp,64 - mov rax,QWORD[120+r8] - mov rbx,QWORD[248+r8] + mov rax,QWORD[120+r8] ; pull context->Rax + mov rbx,QWORD[248+r8] ; pull context->Rip lea r10,[$L$prologue] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip<.Lprologue jb NEAR $L$in_prologue - mov rax,QWORD[152+r8] + mov rax,QWORD[152+r8] ; pull context->Rsp lea r10,[$L$epilogue] - cmp rbx,r10 + cmp rbx,r10 ; context->Rip>=.Lepilogue jae NEAR $L$in_prologue lea rax,[40+rax] @@ -740,39 +740,39 @@ mov r12,QWORD[((-24))+rax] mov r14,QWORD[((-32))+rax] mov r15,QWORD[((-40))+rax] - mov QWORD[144+r8],rbx - mov QWORD[160+r8],rbp - mov QWORD[216+r8],r12 - mov QWORD[232+r8],r14 - mov QWORD[240+r8],r15 + mov QWORD[144+r8],rbx ; restore context->Rbx + mov QWORD[160+r8],rbp ; restore context->Rbp + mov QWORD[216+r8],r12 ; restore context->R12 + mov QWORD[232+r8],r14 ; restore context->R14 + mov QWORD[240+r8],r15 ; restore context->R15 $L$in_prologue: mov rdi,QWORD[8+rax] mov rsi,QWORD[16+rax] - mov QWORD[152+r8],rax - mov QWORD[168+r8],rsi - mov QWORD[176+r8],rdi + mov QWORD[152+r8],rax ; restore context->Rsp + mov QWORD[168+r8],rsi ; restore context->Rsi + mov QWORD[176+r8],rdi ; restore context->Rdi - mov rdi,QWORD[40+r9] - mov rsi,r8 - mov ecx,154 - DD 0xa548f3fc + mov rdi,QWORD[40+r9] ; disp->ContextRecord + mov rsi,r8 ; context + mov ecx,154 ; sizeof(CONTEXT) + DD 0xa548f3fc ; cld; rep movsq mov rsi,r9 - xor rcx,rcx - mov rdx,QWORD[8+rsi] - mov r8,QWORD[rsi] - mov r9,QWORD[16+rsi] - mov r10,QWORD[40+rsi] - lea r11,[56+rsi] - lea r12,[24+rsi] - mov QWORD[32+rsp],r10 - mov QWORD[40+rsp],r11 - mov QWORD[48+rsp],r12 - mov QWORD[56+rsp],rcx + xor rcx,rcx ; arg1, UNW_FLAG_NHANDLER + mov rdx,QWORD[8+rsi] ; arg2, disp->ImageBase + mov r8,QWORD[rsi] ; arg3, disp->ControlPc + mov r9,QWORD[16+rsi] ; arg4, disp->FunctionEntry + mov r10,QWORD[40+rsi] ; disp->ContextRecord + lea r11,[56+rsi] ; &disp->HandlerData + lea r12,[24+rsi] ; &disp->EstablisherFrame + mov QWORD[32+rsp],r10 ; arg5 + mov QWORD[40+rsp],r11 ; arg6 + mov QWORD[48+rsp],r12 ; arg7 + mov QWORD[56+rsp],rcx ; arg8, (NULL) call QWORD[__imp_RtlVirtualUnwind] - mov eax,1 + mov eax,1 ; ExceptionContinueSearch add rsp,64 popfq pop r15
diff --git a/gen/test_support/trampoline-x86_64-apple.S b/gen/test_support/trampoline-x86_64-apple.S index 4d414b0..d98e721 100644 --- a/gen/test_support/trampoline-x86_64-apple.S +++ b/gen/test_support/trampoline-x86_64-apple.S
@@ -6,13 +6,13 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) .text - - - - - - - +// abi_test_trampoline loads callee-saved registers from |state|, calls |func| +// with |argv|, then saves the callee-saved registers into |state|. It returns +// the result of |func|. If |unwind| is non-zero, this function triggers unwind +// instrumentation. +// uint64_t abi_test_trampoline(void (*func)(...), CallerState *state, +// const uint64_t *argv, size_t argc, +// int unwind); .globl _abi_test_trampoline .private_extern _abi_test_trampoline @@ -21,15 +21,15 @@ _CET_ENDBR - - - - - - - - - +// Stack layout: +// 8 bytes - align +// 48 bytes - saved caller registers +// 8 bytes - scratch space +// 8 bytes - saved copy of $unwind (SysV-only) +// 8 bytes - saved copy of $state +// 8 bytes - saved copy of $func +// 8 bytes - if needed for stack alignment +// 8*4 bytes - parameters for $func subq $120,%rsp @@ -59,13 +59,13 @@ movq 24(%rsi),%r13 movq 32(%rsi),%r14 movq 40(%rsi),%r15 - +// Stash $func and $state, so they are available after the call returns. movq %rdi,32(%rsp) movq %rsi,40(%rsp) - - - +// Load parameters. Note this will clobber $argv and $argc, so we can +// only use non-parameter volatile registers. There are three, and they +// are the same between SysV and Win64: %rax, %r10, and %r11. movq %rdx,%r10 movq %rcx,%r11 decq %r11 @@ -97,11 +97,11 @@ decq %r11 js L$args_done - - - - - +// This block should be: +// movq (%r10), %rtmp +// movq %rtmp, (%rax) +// There are no spare registers available, so we spill into the scratch +// space. movq %r11,56(%rsp) movq (%r10),%r11 movq %r11,(%rax) @@ -117,13 +117,13 @@ testq %r10,%r10 jz L$no_unwind - +// Set the trap flag. pushfq orq $0x100,0(%rsp) popfq - - +// Run an instruction to trigger a breakpoint immediately before the +// call. nop .globl _abi_test_unwind_start .private_extern _abi_test_unwind_start @@ -136,11 +136,11 @@ .alt_entry _abi_test_unwind_return _abi_test_unwind_return: - - - +// Clear the trap flag. Note this assumes the trap flag was clear on +// entry. We do not support instrumenting an unwind-instrumented +// |abi_test_trampoline|. pushfq - andq $-0x101,0(%rsp) + andq $-0x101,0(%rsp) // -0x101 is ~0x100 popfq .globl _abi_test_unwind_stop .private_extern _abi_test_unwind_stop @@ -153,7 +153,7 @@ call *%rax L$call_done: - +// Store what $func did our state, so our caller can check. movq 40(%rsp),%rsi movq %rbx,0(%rsi) movq %rbp,8(%rsi) @@ -176,7 +176,7 @@ addq $120,%rsp - +// %rax already contains $func's return value, unmodified. ret @@ -491,9 +491,9 @@ pxor %xmm15,%xmm15 ret - - - +// abi_test_bad_unwind_wrong_register preserves the ABI, but annotates the wrong +// register in unwind metadata. +// void abi_test_bad_unwind_wrong_register(void); .globl _abi_test_bad_unwind_wrong_register .private_extern _abi_test_bad_unwind_wrong_register @@ -504,12 +504,12 @@ _CET_ENDBR pushq %r12 +// This should be %r13 +// This should be %r13 - - - - - +// Windows evaluates epilogs directly in the unwinder, rather than using +// unwind codes. Add a nop so there is one non-epilog point (immediately +// before the nop) where the unwinder can observe the mistake. nop popq %r12 @@ -518,9 +518,9 @@ - - - +// abi_test_bad_unwind_temporary preserves the ABI, but temporarily corrupts the +// storage space for a saved register, breaking unwind. +// void abi_test_bad_unwind_temporary(void); .globl _abi_test_bad_unwind_temporary .private_extern _abi_test_bad_unwind_temporary @@ -538,11 +538,11 @@ movq %r12,%rax incq %rax movq %rax,(%rsp) - - +// Unwinding from here is incorrect. Although %r12 itself has not been +// changed, the unwind codes say to look in (%rsp) instead. movq %r12,(%rsp) - +// Unwinding is now fixed. popq %r12 @@ -551,9 +551,9 @@ - - - +// abi_test_get_and_clear_direction_flag clears the direction flag. If the flag +// was previously set, it returns one. Otherwise, it returns zero. +// int abi_test_get_and_clear_direction_flag(void); .globl _abi_test_get_and_clear_direction_flag .private_extern _abi_test_get_and_clear_direction_flag @@ -568,8 +568,8 @@ ret - - +// abi_test_set_direction_flag sets the direction flag. +// void abi_test_set_direction_flag(void); .globl _abi_test_set_direction_flag .private_extern _abi_test_set_direction_flag
diff --git a/gen/test_support/trampoline-x86_64-linux.S b/gen/test_support/trampoline-x86_64-linux.S index 13d08a3..56ea8b1 100644 --- a/gen/test_support/trampoline-x86_64-linux.S +++ b/gen/test_support/trampoline-x86_64-linux.S
@@ -6,13 +6,13 @@ #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__ELF__) .text - - - - - - - +// abi_test_trampoline loads callee-saved registers from |state|, calls |func| +// with |argv|, then saves the callee-saved registers into |state|. It returns +// the result of |func|. If |unwind| is non-zero, this function triggers unwind +// instrumentation. +// uint64_t abi_test_trampoline(void (*func)(...), CallerState *state, +// const uint64_t *argv, size_t argc, +// int unwind); .globl abi_test_trampoline .hidden abi_test_trampoline .type abi_test_trampoline, @function @@ -21,15 +21,15 @@ .cfi_startproc _CET_ENDBR - - - - - - - - - +// Stack layout: +// 8 bytes - align +// 48 bytes - saved caller registers +// 8 bytes - scratch space +// 8 bytes - saved copy of $unwind (SysV-only) +// 8 bytes - saved copy of $state +// 8 bytes - saved copy of $func +// 8 bytes - if needed for stack alignment +// 8*4 bytes - parameters for $func subq $120,%rsp .cfi_adjust_cfa_offset 120 @@ -59,13 +59,13 @@ movq 24(%rsi),%r13 movq 32(%rsi),%r14 movq 40(%rsi),%r15 - +// Stash $func and $state, so they are available after the call returns. movq %rdi,32(%rsp) movq %rsi,40(%rsp) - - - +// Load parameters. Note this will clobber $argv and $argc, so we can +// only use non-parameter volatile registers. There are three, and they +// are the same between SysV and Win64: %rax, %r10, and %r11. movq %rdx,%r10 movq %rcx,%r11 decq %r11 @@ -97,11 +97,11 @@ decq %r11 js .Largs_done - - - - - +// This block should be: +// movq (%r10), %rtmp +// movq %rtmp, (%rax) +// There are no spare registers available, so we spill into the scratch +// space. movq %r11,56(%rsp) movq (%r10),%r11 movq %r11,(%rax) @@ -117,13 +117,13 @@ testq %r10,%r10 jz .Lno_unwind - +// Set the trap flag. pushfq orq $0x100,0(%rsp) popfq - - +// Run an instruction to trigger a breakpoint immediately before the +// call. nop .globl abi_test_unwind_start .hidden abi_test_unwind_start @@ -134,11 +134,11 @@ .hidden abi_test_unwind_return abi_test_unwind_return: - - - +// Clear the trap flag. Note this assumes the trap flag was clear on +// entry. We do not support instrumenting an unwind-instrumented +// |abi_test_trampoline|. pushfq - andq $-0x101,0(%rsp) + andq $-0x101,0(%rsp) // -0x101 is ~0x100 popfq .globl abi_test_unwind_stop .hidden abi_test_unwind_stop @@ -150,7 +150,7 @@ call *%rax .Lcall_done: - +// Store what $func did our state, so our caller can check. movq 40(%rsp),%rsi movq %rbx,0(%rsi) movq %rbp,8(%rsi) @@ -173,7 +173,7 @@ addq $120,%rsp .cfi_adjust_cfa_offset -120 - +// %rax already contains $func's return value, unmodified. ret .cfi_endproc @@ -457,9 +457,9 @@ pxor %xmm15,%xmm15 ret .size abi_test_clobber_xmm15,.-abi_test_clobber_xmm15 - - - +// abi_test_bad_unwind_wrong_register preserves the ABI, but annotates the wrong +// register in unwind metadata. +// void abi_test_bad_unwind_wrong_register(void); .globl abi_test_bad_unwind_wrong_register .hidden abi_test_bad_unwind_wrong_register .type abi_test_bad_unwind_wrong_register, @function @@ -470,12 +470,12 @@ _CET_ENDBR pushq %r12 .cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-16 +.cfi_offset %r13,-16 // This should be %r13 +// This should be %r13 - - - - +// Windows evaluates epilogs directly in the unwinder, rather than using +// unwind codes. Add a nop so there is one non-epilog point (immediately +// before the nop) where the unwinder can observe the mistake. nop popq %r12 .cfi_adjust_cfa_offset -8 @@ -485,9 +485,9 @@ .cfi_endproc .size abi_test_bad_unwind_wrong_register,.-abi_test_bad_unwind_wrong_register - - - +// abi_test_bad_unwind_temporary preserves the ABI, but temporarily corrupts the +// storage space for a saved register, breaking unwind. +// void abi_test_bad_unwind_temporary(void); .globl abi_test_bad_unwind_temporary .hidden abi_test_bad_unwind_temporary .type abi_test_bad_unwind_temporary, @function @@ -505,11 +505,11 @@ movq %r12,%rax incq %rax movq %rax,(%rsp) - - +// Unwinding from here is incorrect. Although %r12 itself has not been +// changed, the unwind codes say to look in (%rsp) instead. movq %r12,(%rsp) - +// Unwinding is now fixed. popq %r12 .cfi_adjust_cfa_offset -8 @@ -519,9 +519,9 @@ .size abi_test_bad_unwind_temporary,.-abi_test_bad_unwind_temporary - - - +// abi_test_get_and_clear_direction_flag clears the direction flag. If the flag +// was previously set, it returns one. Otherwise, it returns zero. +// int abi_test_get_and_clear_direction_flag(void); .globl abi_test_get_and_clear_direction_flag .hidden abi_test_get_and_clear_direction_flag .type abi_test_get_and_clear_direction_flag, @function @@ -535,8 +535,8 @@ ret .size abi_test_get_and_clear_direction_flag,.-abi_test_get_and_clear_direction_flag - - +// abi_test_set_direction_flag sets the direction flag. +// void abi_test_set_direction_flag(void); .globl abi_test_set_direction_flag .hidden abi_test_set_direction_flag .type abi_test_set_direction_flag, @function
diff --git a/gen/test_support/trampoline-x86_64-win.asm b/gen/test_support/trampoline-x86_64-win.asm index ca2fb87..9ffbd6a 100644 --- a/gen/test_support/trampoline-x86_64-win.asm +++ b/gen/test_support/trampoline-x86_64-win.asm
@@ -14,13 +14,13 @@ section .text code align=64 - - - - - - - +; abi_test_trampoline loads callee-saved registers from |state|, calls |func| +; with |argv|, then saves the callee-saved registers into |state|. It returns +; the result of |func|. If |unwind| is non-zero, this function triggers unwind +; instrumentation. +; uint64_t abi_test_trampoline(void (*func)(...), CallerState *state, +; const uint64_t *argv, size_t argc, +; int unwind); global abi_test_trampoline ALIGN 16 @@ -28,15 +28,15 @@ $L$SEH_begin_abi_test_trampoline_1: _CET_ENDBR - - - - - - - - - +; Stack layout: +; 8 bytes - align +; 224 bytes - saved caller registers +; 8 bytes - scratch space +; 8 bytes - saved copy of $unwind (SysV-only) +; 8 bytes - saved copy of $state +; 8 bytes - saved copy of $func +; 8 bytes - if needed for stack alignment +; 8*10 bytes - parameters for $func sub rsp,344 $L$SEH_prologue_abi_test_trampoline_2: @@ -113,13 +113,13 @@ movdqa xmm13,XMMWORD[176+rdx] movdqa xmm14,XMMWORD[192+rdx] movdqa xmm15,XMMWORD[208+rdx] - +; Stash $func and $state, so they are available after the call returns. mov QWORD[88+rsp],rcx mov QWORD[96+rsp],rdx - - - +; Load parameters. Note this will clobber $argv and $argc, so we can +; only use non-parameter volatile registers. There are three, and they +; are the same between SysV and Win64: %rax, %r10, and %r11. mov r10,r8 mov r11,r9 dec r11 @@ -143,11 +143,11 @@ dec r11 js NEAR $L$args_done - - - - - +; This block should be: +; movq (%r10), %rtmp +; movq %rtmp, (%rax) +; There are no spare registers available, so we spill into the scratch +; space. mov QWORD[104+rsp],r11 mov r11,QWORD[r10] mov QWORD[rax],r11 @@ -163,13 +163,13 @@ test r10,r10 jz NEAR $L$no_unwind - +; Set the trap flag. pushfq or QWORD[rsp],0x100 popfq - - +; Run an instruction to trigger a breakpoint immediately before the +; call. nop global abi_test_unwind_start abi_test_unwind_start: @@ -178,11 +178,11 @@ global abi_test_unwind_return abi_test_unwind_return: - - - +; Clear the trap flag. Note this assumes the trap flag was clear on +; entry. We do not support instrumenting an unwind-instrumented +; |abi_test_trampoline|. pushfq - and QWORD[rsp],-0x101 + and QWORD[rsp],-0x101 ; -0x101 is ~0x100 popfq global abi_test_unwind_stop abi_test_unwind_stop: @@ -193,7 +193,7 @@ call rax $L$call_done: - +; Store what $func did our state, so our caller can check. mov rdx,QWORD[96+rsp] mov QWORD[rdx],rbx mov QWORD[8+rdx],rbp @@ -252,7 +252,7 @@ add rsp,344 - +; %rax already contains $func's return value, unmodified. ret $L$SEH_end_abi_test_trampoline_22: @@ -505,9 +505,9 @@ pxor xmm15,xmm15 ret - - - +; abi_test_bad_unwind_wrong_register preserves the ABI, but annotates the wrong +; register in unwind metadata. +; void abi_test_bad_unwind_wrong_register(void); global abi_test_bad_unwind_wrong_register ALIGN 16 @@ -516,12 +516,12 @@ $L$SEH_begin_abi_test_bad_unwind_wrong_register_1: _CET_ENDBR push r12 - -$L$SEH_prologue_abi_test_bad_unwind_wrong_register_2: +; This should be %r13 +$L$SEH_prologue_abi_test_bad_unwind_wrong_register_2: ; This should be %r13 $L$SEH_endprologue_abi_test_bad_unwind_wrong_register_3: - - - +; Windows evaluates epilogs directly in the unwinder, rather than using +; unwind codes. Add a nop so there is one non-epilog point (immediately +; before the nop) where the unwinder can observe the mistake. nop pop r12 @@ -530,9 +530,9 @@ - - - +; abi_test_bad_unwind_temporary preserves the ABI, but temporarily corrupts the +; storage space for a saved register, breaking unwind. +; void abi_test_bad_unwind_temporary(void); global abi_test_bad_unwind_temporary ALIGN 16 @@ -548,11 +548,11 @@ mov rax,r12 inc rax mov QWORD[rsp],rax - - +; Unwinding from here is incorrect. Although %r12 itself has not been +; changed, the unwind codes say to look in (%rsp) instead. mov QWORD[rsp],r12 - +; Unwinding is now fixed. pop r12 @@ -561,9 +561,9 @@ $L$SEH_end_abi_test_bad_unwind_temporary_4: - - - +; abi_test_get_and_clear_direction_flag clears the direction flag. If the flag +; was previously set, it returns one. Otherwise, it returns zero. +; int abi_test_get_and_clear_direction_flag(void); global abi_test_get_and_clear_direction_flag abi_test_get_and_clear_direction_flag: @@ -576,8 +576,8 @@ ret - - +; abi_test_set_direction_flag sets the direction flag. +; void abi_test_set_direction_flag(void); global abi_test_set_direction_flag abi_test_set_direction_flag: @@ -585,10 +585,10 @@ std ret - - - - +; abi_test_bad_unwind_epilog preserves the ABI, and correctly annotates the +; prolog, but the epilog does not match Win64's rules, breaking unwind during +; the epilog. +; void abi_test_bad_unwind_epilog(void); global abi_test_bad_unwind_epilog ALIGN 16 @@ -600,7 +600,7 @@ nop - +; The epilog should begin here, but the nop makes it invalid. pop r12 nop ret @@ -628,7 +628,7 @@ section .xdata rdata align=8 ALIGN 4 $L$SEH_info_abi_test_trampoline_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_abi_test_trampoline_21-$L$SEH_begin_abi_test_trampoline_1 DB 38 DB 0 @@ -691,7 +691,7 @@ DW 43 $L$SEH_info_abi_test_bad_unwind_wrong_register_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_abi_test_bad_unwind_wrong_register_3-$L$SEH_begin_abi_test_bad_unwind_wrong_register_1 DB 1 DB 0 @@ -700,7 +700,7 @@ DW 0 $L$SEH_info_abi_test_bad_unwind_temporary_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_abi_test_bad_unwind_temporary_3-$L$SEH_begin_abi_test_bad_unwind_temporary_1 DB 1 DB 0 @@ -709,7 +709,7 @@ DW 0 $L$SEH_info_abi_test_bad_unwind_epilog_0: - DB 1 + DB 1 ; version 1, no flags DB $L$SEH_endprologue_abi_test_bad_unwind_epilog_3-$L$SEH_begin_abi_test_bad_unwind_epilog_1 DB 1 DB 0