blob: 62412377221b8872b78f562b896e05241fbb44a8 [file] [log] [blame]
#!/usr/bin/env perl
# Copyright (c) 2017, Shay Gueron.
# Copyright (c) 2017, Google Inc.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
use warnings FATAL => 'all';
$flavour = shift;
$output = shift;
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
die "can't locate x86_64-xlate.pl";
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;
$code.=<<___;
.section .rodata
.align 16
one:
.quad 1,0
two:
.quad 2,0
three:
.quad 3,0
four:
.quad 4,0
five:
.quad 5,0
six:
.quad 6,0
seven:
.quad 7,0
eight:
.quad 8,0
OR_MASK:
.long 0x00000000,0x00000000,0x00000000,0x80000000
poly:
.quad 0x1, 0xc200000000000000
mask:
.long 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d
con1:
.long 1,1,1,1
con2:
.long 0x1b,0x1b,0x1b,0x1b
con3:
.byte -1,-1,-1,-1,-1,-1,-1,-1,4,5,6,7,4,5,6,7
and_mask:
.long 0,0xffffffff, 0xffffffff, 0xffffffff
___
$code.=<<___;
.text
___
sub gfmul {
#########################
# a = T
# b = TMP0 - remains unchanged
# res = T
# uses also TMP1,TMP2,TMP3,TMP4
# __m128i GFMUL(__m128i A, __m128i B);
my $T = "%xmm0";
my $TMP0 = "%xmm1";
my $TMP1 = "%xmm2";
my $TMP2 = "%xmm3";
my $TMP3 = "%xmm4";
my $TMP4 = "%xmm5";
$code.=<<___;
.type GFMUL,\@abi-omnipotent
.align 16
GFMUL:
.cfi_startproc
vpclmulqdq \$0x00, $TMP0, $T, $TMP1
vpclmulqdq \$0x11, $TMP0, $T, $TMP4
vpclmulqdq \$0x10, $TMP0, $T, $TMP2
vpclmulqdq \$0x01, $TMP0, $T, $TMP3
vpxor $TMP3, $TMP2, $TMP2
vpslldq \$8, $TMP2, $TMP3
vpsrldq \$8, $TMP2, $TMP2
vpxor $TMP3, $TMP1, $TMP1
vpxor $TMP2, $TMP4, $TMP4
vpclmulqdq \$0x10, poly(%rip), $TMP1, $TMP2
vpshufd \$78, $TMP1, $TMP3
vpxor $TMP3, $TMP2, $TMP1
vpclmulqdq \$0x10, poly(%rip), $TMP1, $TMP2
vpshufd \$78, $TMP1, $TMP3
vpxor $TMP3, $TMP2, $TMP1
vpxor $TMP4, $TMP1, $T
ret
.cfi_endproc
.size GFMUL, .-GFMUL
___
}
gfmul();
sub aesgcmsiv_htable_init {
# aesgcmsiv_htable_init writes an eight-entry table of powers of |H| to
# |out_htable|.
# void aesgcmsiv_htable_init(uint8_t out_htable[16*8], uint8_t *H);
my $Htbl = "%rdi";
my $H = "%rsi";
my $T = "%xmm0";
my $TMP0 = "%xmm1";
$code.=<<___;
.globl aesgcmsiv_htable_init
.type aesgcmsiv_htable_init,\@function,2
.align 16
aesgcmsiv_htable_init:
.cfi_startproc
_CET_ENDBR
vmovdqa ($H), $T
vmovdqa $T, $TMP0
vmovdqa $T, ($Htbl) # H
call GFMUL
vmovdqa $T, 16($Htbl) # H^2
call GFMUL
vmovdqa $T, 32($Htbl) # H^3
call GFMUL
vmovdqa $T, 48($Htbl) # H^4
call GFMUL
vmovdqa $T, 64($Htbl) # H^5
call GFMUL
vmovdqa $T, 80($Htbl) # H^6
call GFMUL
vmovdqa $T, 96($Htbl) # H^7
call GFMUL
vmovdqa $T, 112($Htbl) # H^8
ret
.cfi_endproc
.size aesgcmsiv_htable_init, .-aesgcmsiv_htable_init
___
}
aesgcmsiv_htable_init();
sub aesgcmsiv_htable6_init {
# aesgcmsiv_htable6_init writes a six-entry table of powers of |H| to
# |out_htable|.
# void aesgcmsiv_htable6_init(uint8_t out_htable[16*6], uint8_t *H);
#
my $Htbl = "%rdi";
my $H = "%rsi";
my $T = "%xmm0";
my $TMP0 = "%xmm1";
$code.=<<___;
.globl aesgcmsiv_htable6_init
.type aesgcmsiv_htable6_init,\@function,2
.align 16
aesgcmsiv_htable6_init:
.cfi_startproc
_CET_ENDBR
vmovdqa ($H), $T
vmovdqa $T, $TMP0
vmovdqa $T, ($Htbl) # H
call GFMUL
vmovdqa $T, 16($Htbl) # H^2
call GFMUL
vmovdqa $T, 32($Htbl) # H^3
call GFMUL
vmovdqa $T, 48($Htbl) # H^4
call GFMUL
vmovdqa $T, 64($Htbl) # H^5
call GFMUL
vmovdqa $T, 80($Htbl) # H^6
ret
.cfi_endproc
.size aesgcmsiv_htable6_init, .-aesgcmsiv_htable6_init
___
}
aesgcmsiv_htable6_init();
sub aesgcmsiv_htable_polyval {
# void aesgcmsiv_htable_polyval(uint8_t Htbl[16*8], uint8_t *MSG, uint64_t LEN, uint8_t *T);
# parameter 1: %rdi Htable - pointer to Htable
# parameter 2: %rsi INp - pointer to input
# parameter 3: %rdx LEN - length of BUFFER in bytes
# parameter 4: %rcx T - pointer to POLYVAL output
my $DATA = "%xmm0";
my $hlp0 = "%r11";
my $Htbl = "%rdi";
my $inp = "%rsi";
my $len = "%rdx";
my $TMP0 = "%xmm3";
my $TMP1 = "%xmm4";
my $TMP2 = "%xmm5";
my $TMP3 = "%xmm6";
my $TMP4 = "%xmm7";
my $Tp = "%rcx";
my $T = "%xmm1";
my $Xhi = "%xmm9";
my $SCHOOLBOOK_AAD = sub {
my ($i)=@_;
return <<___;
vpclmulqdq \$0x01, ${\eval(16*$i)}($Htbl), $DATA, $TMP3
vpxor $TMP3, $TMP2, $TMP2
vpclmulqdq \$0x00, ${\eval(16*$i)}($Htbl), $DATA, $TMP3
vpxor $TMP3, $TMP0, $TMP0
vpclmulqdq \$0x11, ${\eval(16*$i)}($Htbl), $DATA, $TMP3
vpxor $TMP3, $TMP1, $TMP1
vpclmulqdq \$0x10, ${\eval(16*$i)}($Htbl), $DATA, $TMP3
vpxor $TMP3, $TMP2, $TMP2
___
};
$code.=<<___;
.globl aesgcmsiv_htable_polyval
.type aesgcmsiv_htable_polyval,\@function,4
.align 16
aesgcmsiv_htable_polyval:
.cfi_startproc
_CET_ENDBR
test $len, $len
jnz .Lhtable_polyval_start
ret
.Lhtable_polyval_start:
vzeroall
# We hash 8 blocks each iteration. If the total number of blocks is not a
# multiple of 8, we first hash the leading n%8 blocks.
movq $len, $hlp0
andq \$127, $hlp0
jz .Lhtable_polyval_no_prefix
vpxor $Xhi, $Xhi, $Xhi
vmovdqa ($Tp), $T
sub $hlp0, $len
sub \$16, $hlp0
# hash first prefix block
vmovdqu ($inp), $DATA
vpxor $T, $DATA, $DATA
vpclmulqdq \$0x01, ($Htbl,$hlp0), $DATA, $TMP2
vpclmulqdq \$0x00, ($Htbl,$hlp0), $DATA, $TMP0
vpclmulqdq \$0x11, ($Htbl,$hlp0), $DATA, $TMP1
vpclmulqdq \$0x10, ($Htbl,$hlp0), $DATA, $TMP3
vpxor $TMP3, $TMP2, $TMP2
lea 16($inp), $inp
test $hlp0, $hlp0
jnz .Lhtable_polyval_prefix_loop
jmp .Lhtable_polyval_prefix_complete
# hash remaining prefix bocks (up to 7 total prefix blocks)
.align 64
.Lhtable_polyval_prefix_loop:
sub \$16, $hlp0
vmovdqu ($inp), $DATA # next data block
vpclmulqdq \$0x00, ($Htbl,$hlp0), $DATA, $TMP3
vpxor $TMP3, $TMP0, $TMP0
vpclmulqdq \$0x11, ($Htbl,$hlp0), $DATA, $TMP3
vpxor $TMP3, $TMP1, $TMP1
vpclmulqdq \$0x01, ($Htbl,$hlp0), $DATA, $TMP3
vpxor $TMP3, $TMP2, $TMP2
vpclmulqdq \$0x10, ($Htbl,$hlp0), $DATA, $TMP3
vpxor $TMP3, $TMP2, $TMP2
test $hlp0, $hlp0
lea 16($inp), $inp
jnz .Lhtable_polyval_prefix_loop
.Lhtable_polyval_prefix_complete:
vpsrldq \$8, $TMP2, $TMP3
vpslldq \$8, $TMP2, $TMP2
vpxor $TMP3, $TMP1, $Xhi
vpxor $TMP2, $TMP0, $T
jmp .Lhtable_polyval_main_loop
.Lhtable_polyval_no_prefix:
# At this point we know the number of blocks is a multiple of 8. However,
# the reduction in the main loop includes a multiplication by x^(-128). In
# order to counter this, the existing tag needs to be multipled by x^128.
# In practice, this just means that it is loaded into $Xhi, not $T.
vpxor $T, $T, $T
vmovdqa ($Tp), $Xhi
.align 64
.Lhtable_polyval_main_loop:
sub \$0x80, $len
jb .Lhtable_polyval_out
vmovdqu 16*7($inp), $DATA # Ii
vpclmulqdq \$0x01, ($Htbl), $DATA, $TMP2
vpclmulqdq \$0x00, ($Htbl), $DATA, $TMP0
vpclmulqdq \$0x11, ($Htbl), $DATA, $TMP1
vpclmulqdq \$0x10, ($Htbl), $DATA, $TMP3
vpxor $TMP3, $TMP2, $TMP2
#########################################################
vmovdqu 16*6($inp), $DATA
${\$SCHOOLBOOK_AAD->(1)}
#########################################################
vmovdqu 16*5($inp), $DATA
vpclmulqdq \$0x10, poly(%rip), $T, $TMP4 # reduction stage 1a
vpalignr \$8, $T, $T, $T
${\$SCHOOLBOOK_AAD->(2)}
vpxor $TMP4, $T, $T # reduction stage 1b
#########################################################
vmovdqu 16*4($inp), $DATA
${\$SCHOOLBOOK_AAD->(3)}
#########################################################
vmovdqu 16*3($inp), $DATA
vpclmulqdq \$0x10, poly(%rip), $T, $TMP4 # reduction stage 2a
vpalignr \$8, $T, $T, $T
${\$SCHOOLBOOK_AAD->(4)}
vpxor $TMP4, $T, $T # reduction stage 2b
#########################################################
vmovdqu 16*2($inp), $DATA
${\$SCHOOLBOOK_AAD->(5)}
vpxor $Xhi, $T, $T # reduction finalize
#########################################################
vmovdqu 16*1($inp), $DATA
${\$SCHOOLBOOK_AAD->(6)}
#########################################################
vmovdqu 16*0($inp), $DATA
vpxor $T, $DATA, $DATA
${\$SCHOOLBOOK_AAD->(7)}
#########################################################
vpsrldq \$8, $TMP2, $TMP3
vpslldq \$8, $TMP2, $TMP2
vpxor $TMP3, $TMP1, $Xhi
vpxor $TMP2, $TMP0, $T
lea 16*8($inp), $inp
jmp .Lhtable_polyval_main_loop
#########################################################
.Lhtable_polyval_out:
vpclmulqdq \$0x10, poly(%rip), $T, $TMP3
vpalignr \$8, $T, $T, $T
vpxor $TMP3, $T, $T
vpclmulqdq \$0x10, poly(%rip), $T, $TMP3
vpalignr \$8, $T, $T, $T
vpxor $TMP3, $T, $T
vpxor $Xhi, $T, $T
vmovdqu $T, ($Tp)
vzeroupper
ret
.cfi_endproc
.size aesgcmsiv_htable_polyval,.-aesgcmsiv_htable_polyval
___
}
aesgcmsiv_htable_polyval();
sub aesgcmsiv_polyval_horner {
#void aesgcmsiv_polyval_horner(unsigned char T[16], // output
# const unsigned char* H, // H
# unsigned char* BUF, // Buffer
# unsigned int blocks); // Len2
#
# parameter 1: %rdi T - pointers to POLYVAL output
# parameter 2: %rsi Hp - pointer to H (user key)
# parameter 3: %rdx INp - pointer to input
# parameter 4: %rcx L - total number of blocks in input BUFFER
#
my $T = "%rdi";
my $Hp = "%rsi";
my $INp = "%rdx";
my $L = "%rcx";
my $LOC = "%r10";
my $LEN = "%eax";
my $H = "%xmm1";
my $RES = "%xmm0";
$code.=<<___;
.globl aesgcmsiv_polyval_horner
.type aesgcmsiv_polyval_horner,\@function,4
.align 16
aesgcmsiv_polyval_horner:
.cfi_startproc
_CET_ENDBR
test $L, $L
jnz .Lpolyval_horner_start
ret
.Lpolyval_horner_start:
# We will start with L GFMULS for POLYVAL(BIG_BUFFER)
# RES = GFMUL(RES, H)
xorq $LOC, $LOC
shlq \$4, $L # L contains number of bytes to process
vmovdqa ($Hp), $H
vmovdqa ($T), $RES
.Lpolyval_horner_loop:
vpxor ($INp,$LOC), $RES, $RES # RES = RES + Xi
call GFMUL # RES = RES * H
add \$16, $LOC
cmp $LOC, $L
jne .Lpolyval_horner_loop
# calculation of T is complete. RES=T
vmovdqa $RES, ($T)
ret
.cfi_endproc
.size aesgcmsiv_polyval_horner,.-aesgcmsiv_polyval_horner
___
}
aesgcmsiv_polyval_horner();
# void aes128gcmsiv_aes_ks(const uint8_t *key, uint8_t *out_expanded_key);
# parameter 1: %rdi
# parameter 2: %rsi
$code.=<<___;
.globl aes128gcmsiv_aes_ks
.type aes128gcmsiv_aes_ks,\@function,2
.align 16
aes128gcmsiv_aes_ks:
.cfi_startproc
_CET_ENDBR
vmovdqu (%rdi), %xmm1 # xmm1 = user key
vmovdqa %xmm1, (%rsi) # rsi points to output
vmovdqa con1(%rip), %xmm0
vmovdqa mask(%rip), %xmm15
movq \$8, %rax
.Lks128_loop:
addq \$16, %rsi # rsi points for next key
subq \$1, %rax
vpshufb %xmm15, %xmm1, %xmm2 # xmm2 = shuffled user key
vaesenclast %xmm0, %xmm2, %xmm2
vpslld \$1, %xmm0, %xmm0
vpslldq \$4, %xmm1, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpslldq \$4, %xmm3, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpslldq \$4, %xmm3, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vmovdqa %xmm1, (%rsi)
jne .Lks128_loop
vmovdqa con2(%rip), %xmm0
vpshufb %xmm15, %xmm1, %xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpslld \$1, %xmm0, %xmm0
vpslldq \$4, %xmm1, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpslldq \$4, %xmm3, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpslldq \$4, %xmm3, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vmovdqa %xmm1, 16(%rsi)
vpshufb %xmm15, %xmm1, %xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpslldq \$4, %xmm1, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpslldq \$4, %xmm3, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpslldq \$4, %xmm3, %xmm3
vpxor %xmm3, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vmovdqa %xmm1, 32(%rsi)
ret
.cfi_endproc
.size aes128gcmsiv_aes_ks,.-aes128gcmsiv_aes_ks
___
# void aes256gcmsiv_aes_ks(const uint8_t *key, uint8_t *out_expanded_key);
# parameter 1: %rdi
# parameter 2: %rsi
$code.=<<___;
.globl aes256gcmsiv_aes_ks
.type aes256gcmsiv_aes_ks,\@function,2
.align 16
aes256gcmsiv_aes_ks:
.cfi_startproc
_CET_ENDBR
vmovdqu (%rdi), %xmm1
vmovdqu 16(%rdi), %xmm3
vmovdqa %xmm1, (%rsi)
vmovdqa %xmm3, 16(%rsi)
vmovdqa con1(%rip), %xmm0
vmovdqa mask(%rip), %xmm15
vpxor %xmm14, %xmm14, %xmm14
mov \$6, %rax
.Lks256_loop:
add \$32, %rsi
subq \$1, %rax
vpshufb %xmm15, %xmm3, %xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpslld \$1, %xmm0, %xmm0
vpsllq \$32, %xmm1, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpshufb con3(%rip), %xmm1, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vmovdqa %xmm1, (%rsi)
vpshufd \$0xff, %xmm1, %xmm2
vaesenclast %xmm14, %xmm2, %xmm2
vpsllq \$32, %xmm3, %xmm4
vpxor %xmm4, %xmm3, %xmm3
vpshufb con3(%rip), %xmm3, %xmm4
vpxor %xmm4, %xmm3, %xmm3
vpxor %xmm2, %xmm3, %xmm3
vmovdqa %xmm3, 16(%rsi)
jne .Lks256_loop
vpshufb %xmm15, %xmm3, %xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpsllq \$32, %xmm1, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpshufb con3(%rip), %xmm1, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vmovdqa %xmm1, 32(%rsi)
ret
.cfi_endproc
___
sub aes128gcmsiv_aes_ks_enc_x1 {
my $KS1_REGA = "%xmm1";
my $KS1_REGB = "%xmm2";
my $BLOCK1 = "%xmm4";
my $AUXREG = "%xmm3";
my $KS_BLOCK = sub {
my ($reg, $reg2, $auxReg) = @_;
return <<___;
vpsllq \$32, $reg, $auxReg #!!saving mov instruction to xmm3
vpxor $auxReg, $reg, $reg
vpshufb con3(%rip), $reg, $auxReg
vpxor $auxReg, $reg, $reg
vpxor $reg2, $reg, $reg
___
};
my $round = sub {
my ($i, $j) = @_;
return <<___;
vpshufb %xmm15, %xmm1, %xmm2 #!!saving mov instruction to xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpslld \$1, %xmm0, %xmm0
${\$KS_BLOCK->($KS1_REGA, $KS1_REGB, $AUXREG)}
vaesenc %xmm1, $BLOCK1, $BLOCK1
vmovdqa %xmm1, ${\eval(16*$i)}($j)
___
};
my $roundlast = sub {
my ($i, $j) = @_;
return <<___;
vpshufb %xmm15, %xmm1, %xmm2 #!!saving mov instruction to xmm2
vaesenclast %xmm0, %xmm2, %xmm2
${\$KS_BLOCK->($KS1_REGA, $KS1_REGB, $AUXREG)}
vaesenclast %xmm1, $BLOCK1, $BLOCK1
vmovdqa %xmm1, ${\eval(16*$i)}($j)
___
};
# parameter 1: %rdi Pointer to PT
# parameter 2: %rsi Pointer to CT
# parameter 4: %rdx Pointer to keys
# parameter 5: %rcx Pointer to initial key
$code.=<<___;
.globl aes128gcmsiv_aes_ks_enc_x1
.type aes128gcmsiv_aes_ks_enc_x1,\@function,4
.align 16
aes128gcmsiv_aes_ks_enc_x1:
.cfi_startproc
_CET_ENDBR
vmovdqa (%rcx), %xmm1 # xmm1 = first 16 bytes of random key
vmovdqa 0*16(%rdi), $BLOCK1
vmovdqa %xmm1, (%rdx) # KEY[0] = first 16 bytes of random key
vpxor %xmm1, $BLOCK1, $BLOCK1
vmovdqa con1(%rip), %xmm0 # xmm0 = 1,1,1,1
vmovdqa mask(%rip), %xmm15 # xmm15 = mask
${\$round->(1, "%rdx")}
${\$round->(2, "%rdx")}
${\$round->(3, "%rdx")}
${\$round->(4, "%rdx")}
${\$round->(5, "%rdx")}
${\$round->(6, "%rdx")}
${\$round->(7, "%rdx")}
${\$round->(8, "%rdx")}
vmovdqa con2(%rip), %xmm0
${\$round->(9, "%rdx")}
${\$roundlast->(10, "%rdx")}
vmovdqa $BLOCK1, 0*16(%rsi)
ret
.cfi_endproc
.size aes128gcmsiv_aes_ks_enc_x1,.-aes128gcmsiv_aes_ks_enc_x1
___
}
aes128gcmsiv_aes_ks_enc_x1();
sub aes128gcmsiv_kdf {
my $BLOCK1 = "%xmm9";
my $BLOCK2 = "%xmm10";
my $BLOCK3 = "%xmm11";
my $BLOCK4 = "%xmm12";
my $BLOCK5 = "%xmm13";
my $BLOCK6 = "%xmm14";
my $ONE = "%xmm13";
my $KSp = "%rdx";
my $STATE_1 = "%xmm1";
my $enc_roundx4 = sub {
my ($i, $j) = @_;
return <<___;
vmovdqa ${\eval($i*16)}(%rdx), $j
vaesenc $j, $BLOCK1, $BLOCK1
vaesenc $j, $BLOCK2, $BLOCK2
vaesenc $j, $BLOCK3, $BLOCK3
vaesenc $j, $BLOCK4, $BLOCK4
___
};
my $enc_roundlastx4 = sub {
my ($i, $j) = @_;
return <<___;
vmovdqa ${\eval($i*16)}(%rdx), $j
vaesenclast $j, $BLOCK1, $BLOCK1
vaesenclast $j, $BLOCK2, $BLOCK2
vaesenclast $j, $BLOCK3, $BLOCK3
vaesenclast $j, $BLOCK4, $BLOCK4
___
};
# void aes128gcmsiv_kdf(const uint8_t nonce[16],
# uint8_t *out_key_material,
# const uint8_t *key_schedule);
$code.=<<___;
.globl aes128gcmsiv_kdf
.type aes128gcmsiv_kdf,\@function,3
.align 16
aes128gcmsiv_kdf:
.cfi_startproc
_CET_ENDBR
# parameter 1: %rdi Pointer to NONCE
# parameter 2: %rsi Pointer to CT
# parameter 4: %rdx Pointer to keys
vmovdqa (%rdx), %xmm1 # xmm1 = first 16 bytes of random key
vmovdqa 0*16(%rdi), $BLOCK1
vmovdqa and_mask(%rip), $BLOCK4
vmovdqa one(%rip), $ONE
vpshufd \$0x90, $BLOCK1, $BLOCK1
vpand $BLOCK4, $BLOCK1, $BLOCK1
vpaddd $ONE, $BLOCK1, $BLOCK2
vpaddd $ONE, $BLOCK2, $BLOCK3
vpaddd $ONE, $BLOCK3, $BLOCK4
vpxor %xmm1, $BLOCK1, $BLOCK1
vpxor %xmm1, $BLOCK2, $BLOCK2
vpxor %xmm1, $BLOCK3, $BLOCK3
vpxor %xmm1, $BLOCK4, $BLOCK4
${\$enc_roundx4->(1, "%xmm1")}
${\$enc_roundx4->(2, "%xmm2")}
${\$enc_roundx4->(3, "%xmm1")}
${\$enc_roundx4->(4, "%xmm2")}
${\$enc_roundx4->(5, "%xmm1")}
${\$enc_roundx4->(6, "%xmm2")}
${\$enc_roundx4->(7, "%xmm1")}
${\$enc_roundx4->(8, "%xmm2")}
${\$enc_roundx4->(9, "%xmm1")}
${\$enc_roundlastx4->(10, "%xmm2")}
vmovdqa $BLOCK1, 0*16(%rsi)
vmovdqa $BLOCK2, 1*16(%rsi)
vmovdqa $BLOCK3, 2*16(%rsi)
vmovdqa $BLOCK4, 3*16(%rsi)
ret
.cfi_endproc
.size aes128gcmsiv_kdf,.-aes128gcmsiv_kdf
___
}
aes128gcmsiv_kdf();
sub aes128gcmsiv_enc_msg_x4 {
my $CTR1 = "%xmm0";
my $CTR2 = "%xmm1";
my $CTR3 = "%xmm2";
my $CTR4 = "%xmm3";
my $ADDER = "%xmm4";
my $STATE1 = "%xmm5";
my $STATE2 = "%xmm6";
my $STATE3 = "%xmm7";
my $STATE4 = "%xmm8";
my $TMP = "%xmm12";
my $TMP2 = "%xmm13";
my $TMP3 = "%xmm14";
my $IV = "%xmm15";
my $PT = "%rdi";
my $CT = "%rsi";
my $TAG = "%rdx";
my $KS = "%rcx";
my $LEN = "%r8";
my $aes_round = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $TMP
vaesenc $TMP, $STATE1, $STATE1
vaesenc $TMP, $STATE2, $STATE2
vaesenc $TMP, $STATE3, $STATE3
vaesenc $TMP, $STATE4, $STATE4
___
};
my $aes_lastround = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $TMP
vaesenclast $TMP, $STATE1, $STATE1
vaesenclast $TMP, $STATE2, $STATE2
vaesenclast $TMP, $STATE3, $STATE3
vaesenclast $TMP, $STATE4, $STATE4
___
};
# void aes128gcmsiv_enc_msg_x4(unsigned char* PT, unsigned char* CT,
# unsigned char* TAG, unsigned char* KS,
# size_t byte_len);
# parameter 1: %rdi #PT
# parameter 2: %rsi #CT
# parameter 3: %rdx #TAG [127 126 ... 0] IV=[127...32]
# parameter 4: %rcx #KS
# parameter 5: %r8 #LEN MSG_length in bytes
$code.=<<___;
.globl aes128gcmsiv_enc_msg_x4
.type aes128gcmsiv_enc_msg_x4,\@function,5
.align 16
aes128gcmsiv_enc_msg_x4:
.cfi_startproc
_CET_ENDBR
test $LEN, $LEN
jnz .L128_enc_msg_x4_start
ret
.L128_enc_msg_x4_start:
pushq %r12
.cfi_push %r12
pushq %r13
.cfi_push %r13
shrq \$4, $LEN # LEN = num of blocks
movq $LEN, %r10
shlq \$62, %r10
shrq \$62, %r10
# make IV from TAG
vmovdqa ($TAG), $IV
vpor OR_MASK(%rip), $IV, $IV #IV = [1]TAG[126...32][00..00]
vmovdqu four(%rip), $ADDER # Register to increment counters
vmovdqa $IV, $CTR1 # CTR1 = TAG[1][127...32][00..00]
vpaddd one(%rip), $IV, $CTR2 # CTR2 = TAG[1][127...32][00..01]
vpaddd two(%rip), $IV, $CTR3 # CTR3 = TAG[1][127...32][00..02]
vpaddd three(%rip), $IV, $CTR4 # CTR4 = TAG[1][127...32][00..03]
shrq \$2, $LEN
je .L128_enc_msg_x4_check_remainder
subq \$64, $CT
subq \$64, $PT
.L128_enc_msg_x4_loop1:
addq \$64, $CT
addq \$64, $PT
vmovdqa $CTR1, $STATE1
vmovdqa $CTR2, $STATE2
vmovdqa $CTR3, $STATE3
vmovdqa $CTR4, $STATE4
vpxor ($KS), $STATE1, $STATE1
vpxor ($KS), $STATE2, $STATE2
vpxor ($KS), $STATE3, $STATE3
vpxor ($KS), $STATE4, $STATE4
${\$aes_round->(1)}
vpaddd $ADDER, $CTR1, $CTR1
${\$aes_round->(2)}
vpaddd $ADDER, $CTR2, $CTR2
${\$aes_round->(3)}
vpaddd $ADDER, $CTR3, $CTR3
${\$aes_round->(4)}
vpaddd $ADDER, $CTR4, $CTR4
${\$aes_round->(5)}
${\$aes_round->(6)}
${\$aes_round->(7)}
${\$aes_round->(8)}
${\$aes_round->(9)}
${\$aes_lastround->(10)}
# XOR with Plaintext
vpxor 0*16($PT), $STATE1, $STATE1
vpxor 1*16($PT), $STATE2, $STATE2
vpxor 2*16($PT), $STATE3, $STATE3
vpxor 3*16($PT), $STATE4, $STATE4
subq \$1, $LEN
vmovdqu $STATE1, 0*16($CT)
vmovdqu $STATE2, 1*16($CT)
vmovdqu $STATE3, 2*16($CT)
vmovdqu $STATE4, 3*16($CT)
jne .L128_enc_msg_x4_loop1
addq \$64,$CT
addq \$64,$PT
.L128_enc_msg_x4_check_remainder:
cmpq \$0, %r10
je .L128_enc_msg_x4_out
.L128_enc_msg_x4_loop2:
# enc each block separately
# CTR1 is the highest counter (even if no LOOP done)
vmovdqa $CTR1, $STATE1
vpaddd one(%rip), $CTR1, $CTR1 # inc counter
vpxor ($KS), $STATE1, $STATE1
vaesenc 16($KS), $STATE1, $STATE1
vaesenc 32($KS), $STATE1, $STATE1
vaesenc 48($KS), $STATE1, $STATE1
vaesenc 64($KS), $STATE1, $STATE1
vaesenc 80($KS), $STATE1, $STATE1
vaesenc 96($KS), $STATE1, $STATE1
vaesenc 112($KS), $STATE1, $STATE1
vaesenc 128($KS), $STATE1, $STATE1
vaesenc 144($KS), $STATE1, $STATE1
vaesenclast 160($KS), $STATE1, $STATE1
# XOR with plaintext
vpxor ($PT), $STATE1, $STATE1
vmovdqu $STATE1, ($CT)
addq \$16, $PT
addq \$16, $CT
subq \$1, %r10
jne .L128_enc_msg_x4_loop2
.L128_enc_msg_x4_out:
popq %r13
.cfi_pop %r13
popq %r12
.cfi_pop %r12
ret
.cfi_endproc
.size aes128gcmsiv_enc_msg_x4,.-aes128gcmsiv_enc_msg_x4
___
}
aes128gcmsiv_enc_msg_x4();
sub aes128gcmsiv_enc_msg_x8 {
my $STATE1 = "%xmm1";
my $STATE2 = "%xmm2";
my $STATE3 = "%xmm3";
my $STATE4 = "%xmm4";
my $STATE5 = "%xmm5";
my $STATE6 = "%xmm6";
my $STATE7 = "%xmm7";
my $STATE8 = "%xmm8";
my $CTR1 = "%xmm0";
my $CTR2 = "%xmm9";
my $CTR3 = "%xmm10";
my $CTR4 = "%xmm11";
my $CTR5 = "%xmm12";
my $CTR6 = "%xmm13";
my $CTR7 = "%xmm14";
my $SCHED = "%xmm15";
my $TMP1 = "%xmm1";
my $TMP2 = "%xmm2";
my $PT = "%rdi";
my $CT = "%rsi";
my $TAG = "%rdx";
my $KS = "%rcx";
my $LEN = "%r8";
my $aes_round8 = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $SCHED
vaesenc $SCHED, $STATE1, $STATE1
vaesenc $SCHED, $STATE2, $STATE2
vaesenc $SCHED, $STATE3, $STATE3
vaesenc $SCHED, $STATE4, $STATE4
vaesenc $SCHED, $STATE5, $STATE5
vaesenc $SCHED, $STATE6, $STATE6
vaesenc $SCHED, $STATE7, $STATE7
vaesenc $SCHED, $STATE8, $STATE8
___
};
my $aes_lastround8 = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $SCHED
vaesenclast $SCHED, $STATE1, $STATE1
vaesenclast $SCHED, $STATE2, $STATE2
vaesenclast $SCHED, $STATE3, $STATE3
vaesenclast $SCHED, $STATE4, $STATE4
vaesenclast $SCHED, $STATE5, $STATE5
vaesenclast $SCHED, $STATE6, $STATE6
vaesenclast $SCHED, $STATE7, $STATE7
vaesenclast $SCHED, $STATE8, $STATE8
___
};
# void ENC_MSG_x8(unsigned char* PT,
# unsigned char* CT,
# unsigned char* TAG,
# unsigned char* KS,
# size_t byte_len);
# parameter 1: %rdi #PT
# parameter 2: %rsi #CT
# parameter 3: %rdx #TAG [127 126 ... 0] IV=[127...32]
# parameter 4: %rcx #KS
# parameter 5: %r8 #LEN MSG_length in bytes
$code.=<<___;
.globl aes128gcmsiv_enc_msg_x8
.type aes128gcmsiv_enc_msg_x8,\@function,5
.align 16
aes128gcmsiv_enc_msg_x8:
.cfi_startproc
_CET_ENDBR
test $LEN, $LEN
jnz .L128_enc_msg_x8_start
ret
.L128_enc_msg_x8_start:
pushq %r12
.cfi_push %r12
pushq %r13
.cfi_push %r13
pushq %rbp
.cfi_push %rbp
movq %rsp, %rbp
.cfi_def_cfa_register rbp
# Place in stack
subq \$128, %rsp
andq \$-64, %rsp
shrq \$4, $LEN # LEN = num of blocks
movq $LEN, %r10
shlq \$61, %r10
shrq \$61, %r10
# make IV from TAG
vmovdqu ($TAG), $TMP1
vpor OR_MASK(%rip), $TMP1, $TMP1 # TMP1= IV = [1]TAG[126...32][00..00]
# store counter8 in the stack
vpaddd seven(%rip), $TMP1, $CTR1
vmovdqu $CTR1, (%rsp) # CTR8 = TAG[127...32][00..07]
vpaddd one(%rip), $TMP1, $CTR2 # CTR2 = TAG[127...32][00..01]
vpaddd two(%rip), $TMP1, $CTR3 # CTR3 = TAG[127...32][00..02]
vpaddd three(%rip), $TMP1, $CTR4 # CTR4 = TAG[127...32][00..03]
vpaddd four(%rip), $TMP1, $CTR5 # CTR5 = TAG[127...32][00..04]
vpaddd five(%rip), $TMP1, $CTR6 # CTR6 = TAG[127...32][00..05]
vpaddd six(%rip), $TMP1, $CTR7 # CTR7 = TAG[127...32][00..06]
vmovdqa $TMP1, $CTR1 # CTR1 = TAG[127...32][00..00]
shrq \$3, $LEN
je .L128_enc_msg_x8_check_remainder
subq \$128, $CT
subq \$128, $PT
.L128_enc_msg_x8_loop1:
addq \$128, $CT
addq \$128, $PT
vmovdqa $CTR1, $STATE1
vmovdqa $CTR2, $STATE2
vmovdqa $CTR3, $STATE3
vmovdqa $CTR4, $STATE4
vmovdqa $CTR5, $STATE5
vmovdqa $CTR6, $STATE6
vmovdqa $CTR7, $STATE7
# move from stack
vmovdqu (%rsp), $STATE8
vpxor ($KS), $STATE1, $STATE1
vpxor ($KS), $STATE2, $STATE2
vpxor ($KS), $STATE3, $STATE3
vpxor ($KS), $STATE4, $STATE4
vpxor ($KS), $STATE5, $STATE5
vpxor ($KS), $STATE6, $STATE6
vpxor ($KS), $STATE7, $STATE7
vpxor ($KS), $STATE8, $STATE8
${\$aes_round8->(1)}
vmovdqu (%rsp), $CTR7 # deal with CTR8
vpaddd eight(%rip), $CTR7, $CTR7
vmovdqu $CTR7, (%rsp)
${\$aes_round8->(2)}
vpsubd one(%rip), $CTR7, $CTR7
${\$aes_round8->(3)}
vpaddd eight(%rip), $CTR1, $CTR1
${\$aes_round8->(4)}
vpaddd eight(%rip), $CTR2, $CTR2
${\$aes_round8->(5)}
vpaddd eight(%rip), $CTR3, $CTR3
${\$aes_round8->(6)}
vpaddd eight(%rip), $CTR4, $CTR4
${\$aes_round8->(7)}
vpaddd eight(%rip), $CTR5, $CTR5
${\$aes_round8->(8)}
vpaddd eight(%rip), $CTR6, $CTR6
${\$aes_round8->(9)}
${\$aes_lastround8->(10)}
# XOR with Plaintext
vpxor 0*16($PT), $STATE1, $STATE1
vpxor 1*16($PT), $STATE2, $STATE2
vpxor 2*16($PT), $STATE3, $STATE3
vpxor 3*16($PT), $STATE4, $STATE4
vpxor 4*16($PT), $STATE5, $STATE5
vpxor 5*16($PT), $STATE6, $STATE6
vpxor 6*16($PT), $STATE7, $STATE7
vpxor 7*16($PT), $STATE8, $STATE8
dec $LEN
vmovdqu $STATE1, 0*16($CT)
vmovdqu $STATE2, 1*16($CT)
vmovdqu $STATE3, 2*16($CT)
vmovdqu $STATE4, 3*16($CT)
vmovdqu $STATE5, 4*16($CT)
vmovdqu $STATE6, 5*16($CT)
vmovdqu $STATE7, 6*16($CT)
vmovdqu $STATE8, 7*16($CT)
jne .L128_enc_msg_x8_loop1
addq \$128, $CT
addq \$128, $PT
.L128_enc_msg_x8_check_remainder:
cmpq \$0, %r10
je .L128_enc_msg_x8_out
.L128_enc_msg_x8_loop2:
# enc each block separately
# CTR1 is the highest counter (even if no LOOP done)
vmovdqa $CTR1, $STATE1
vpaddd one(%rip), $CTR1, $CTR1 # inc counter
vpxor ($KS), $STATE1, $STATE1
vaesenc 16($KS), $STATE1, $STATE1
vaesenc 32($KS), $STATE1, $STATE1
vaesenc 48($KS), $STATE1, $STATE1
vaesenc 64($KS), $STATE1, $STATE1
vaesenc 80($KS), $STATE1, $STATE1
vaesenc 96($KS), $STATE1, $STATE1
vaesenc 112($KS), $STATE1, $STATE1
vaesenc 128($KS), $STATE1, $STATE1
vaesenc 144($KS), $STATE1, $STATE1
vaesenclast 160($KS), $STATE1, $STATE1
# XOR with Plaintext
vpxor ($PT), $STATE1, $STATE1
vmovdqu $STATE1, ($CT)
addq \$16, $PT
addq \$16, $CT
decq %r10
jne .L128_enc_msg_x8_loop2
.L128_enc_msg_x8_out:
movq %rbp, %rsp
.cfi_def_cfa_register %rsp
popq %rbp
.cfi_pop %rbp
popq %r13
.cfi_pop %r13
popq %r12
.cfi_pop %r12
ret
.cfi_endproc
.size aes128gcmsiv_enc_msg_x8,.-aes128gcmsiv_enc_msg_x8
___
}
aes128gcmsiv_enc_msg_x8();
sub aesgcmsiv_dec {
my ($aes256) = @_;
my $T = "%xmm0";
my $TMP0 = "%xmm1";
my $TMP1 = "%xmm2";
my $TMP2 = "%xmm3";
my $TMP3 = "%xmm4";
my $TMP4 = "%xmm5";
my $TMP5 = "%xmm6";
my $CTR1 = "%xmm7";
my $CTR2 = "%xmm8";
my $CTR3 = "%xmm9";
my $CTR4 = "%xmm10";
my $CTR5 = "%xmm11";
my $CTR6 = "%xmm12";
my $CTR = "%xmm15";
my $CT = "%rdi";
my $PT = "%rsi";
my $POL = "%rdx";
my $Htbl = "%rcx";
my $KS = "%r8";
my $LEN = "%r9";
my $secureBuffer = "%rax";
my $HTABLE_ROUNDS = "%xmm13";
my $labelPrefix = "128";
if ($aes256) {
$labelPrefix = "256";
}
my $aes_round_dec = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $TMP3
vaesenc $TMP3, $CTR1, $CTR1
vaesenc $TMP3, $CTR2, $CTR2
vaesenc $TMP3, $CTR3, $CTR3
vaesenc $TMP3, $CTR4, $CTR4
vaesenc $TMP3, $CTR5, $CTR5
vaesenc $TMP3, $CTR6, $CTR6
___
};
my $aes_lastround_dec = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $TMP3
vaesenclast $TMP3, $CTR1, $CTR1
vaesenclast $TMP3, $CTR2, $CTR2
vaesenclast $TMP3, $CTR3, $CTR3
vaesenclast $TMP3, $CTR4, $CTR4
vaesenclast $TMP3, $CTR5, $CTR5
vaesenclast $TMP3, $CTR6, $CTR6
___
};
my $schoolbook = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16-32)}($secureBuffer), $TMP5
vmovdqu ${\eval($i*16-32)}($Htbl), $HTABLE_ROUNDS
vpclmulqdq \$0x10, $HTABLE_ROUNDS, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
vpclmulqdq \$0x11, $HTABLE_ROUNDS, $TMP5, $TMP3
vpxor $TMP3, $TMP1, $TMP1
vpclmulqdq \$0x00, $HTABLE_ROUNDS, $TMP5, $TMP3
vpxor $TMP3, $TMP2, $TMP2
vpclmulqdq \$0x01, $HTABLE_ROUNDS, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
___
};
if ($aes256) {
$code.=<<___;
.globl aes256gcmsiv_dec
.type aes256gcmsiv_dec,\@function,6
.align 16
aes256gcmsiv_dec:
___
} else {
$code.=<<___;
.globl aes128gcmsiv_dec
.type aes128gcmsiv_dec,\@function,6
.align 16
aes128gcmsiv_dec:
___
}
$code.=<<___;
.cfi_startproc
_CET_ENDBR
test \$~15, $LEN
jnz .L${labelPrefix}_dec_start
ret
.L${labelPrefix}_dec_start:
vzeroupper
vmovdqa ($POL), $T
# The claimed tag is provided after the current calculated tag value.
# CTRBLKs is made from it.
vmovdqu 16($POL), $CTR
vpor OR_MASK(%rip), $CTR, $CTR # CTR = [1]TAG[126...32][00..00]
movq $POL, $secureBuffer
leaq 32($secureBuffer), $secureBuffer
leaq 32($Htbl), $Htbl
andq \$~15, $LEN
# If less then 6 blocks, make singles
cmp \$96, $LEN
jb .L${labelPrefix}_dec_loop2
# Decrypt the first six blocks
sub \$96, $LEN
vmovdqa $CTR, $CTR1
vpaddd one(%rip), $CTR1, $CTR2
vpaddd two(%rip), $CTR1, $CTR3
vpaddd one(%rip), $CTR3, $CTR4
vpaddd two(%rip), $CTR3, $CTR5
vpaddd one(%rip), $CTR5, $CTR6
vpaddd two(%rip), $CTR5, $CTR
vpxor ($KS), $CTR1, $CTR1
vpxor ($KS), $CTR2, $CTR2
vpxor ($KS), $CTR3, $CTR3
vpxor ($KS), $CTR4, $CTR4
vpxor ($KS), $CTR5, $CTR5
vpxor ($KS), $CTR6, $CTR6
${\$aes_round_dec->(1)}
${\$aes_round_dec->(2)}
${\$aes_round_dec->(3)}
${\$aes_round_dec->(4)}
${\$aes_round_dec->(5)}
${\$aes_round_dec->(6)}
${\$aes_round_dec->(7)}
${\$aes_round_dec->(8)}
${\$aes_round_dec->(9)}
___
if ($aes256) {
$code.=<<___;
${\$aes_round_dec->(10)}
${\$aes_round_dec->(11)}
${\$aes_round_dec->(12)}
${\$aes_round_dec->(13)}
${\$aes_lastround_dec->(14)}
___
} else {
$code.=<<___;
${\$aes_lastround_dec->(10)}
___
}
$code.=<<___;
# XOR with CT
vpxor 0*16($CT), $CTR1, $CTR1
vpxor 1*16($CT), $CTR2, $CTR2
vpxor 2*16($CT), $CTR3, $CTR3
vpxor 3*16($CT), $CTR4, $CTR4
vpxor 4*16($CT), $CTR5, $CTR5
vpxor 5*16($CT), $CTR6, $CTR6
vmovdqu $CTR1, 0*16($PT)
vmovdqu $CTR2, 1*16($PT)
vmovdqu $CTR3, 2*16($PT)
vmovdqu $CTR4, 3*16($PT)
vmovdqu $CTR5, 4*16($PT)
vmovdqu $CTR6, 5*16($PT)
addq \$96, $CT
addq \$96, $PT
jmp .L${labelPrefix}_dec_loop1
# Decrypt 6 blocks each time while hashing previous 6 blocks
.align 64
.L${labelPrefix}_dec_loop1:
cmp \$96, $LEN
jb .L${labelPrefix}_dec_finish_96
sub \$96, $LEN
vmovdqa $CTR6, $TMP5
vmovdqa $CTR5, 1*16-32($secureBuffer)
vmovdqa $CTR4, 2*16-32($secureBuffer)
vmovdqa $CTR3, 3*16-32($secureBuffer)
vmovdqa $CTR2, 4*16-32($secureBuffer)
vmovdqa $CTR1, 5*16-32($secureBuffer)
vmovdqa $CTR, $CTR1
vpaddd one(%rip), $CTR1, $CTR2
vpaddd two(%rip), $CTR1, $CTR3
vpaddd one(%rip), $CTR3, $CTR4
vpaddd two(%rip), $CTR3, $CTR5
vpaddd one(%rip), $CTR5, $CTR6
vpaddd two(%rip), $CTR5, $CTR
vmovdqa ($KS), $TMP3
vpxor $TMP3, $CTR1, $CTR1
vpxor $TMP3, $CTR2, $CTR2
vpxor $TMP3, $CTR3, $CTR3
vpxor $TMP3, $CTR4, $CTR4
vpxor $TMP3, $CTR5, $CTR5
vpxor $TMP3, $CTR6, $CTR6
vmovdqu 0*16-32($Htbl), $TMP3
vpclmulqdq \$0x11, $TMP3, $TMP5, $TMP1
vpclmulqdq \$0x00, $TMP3, $TMP5, $TMP2
vpclmulqdq \$0x01, $TMP3, $TMP5, $TMP0
vpclmulqdq \$0x10, $TMP3, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
${\$aes_round_dec->(1)}
${\$schoolbook->(1)}
${\$aes_round_dec->(2)}
${\$schoolbook->(2)}
${\$aes_round_dec->(3)}
${\$schoolbook->(3)}
${\$aes_round_dec->(4)}
${\$schoolbook->(4)}
${\$aes_round_dec->(5)}
${\$aes_round_dec->(6)}
${\$aes_round_dec->(7)}
vmovdqa 5*16-32($secureBuffer), $TMP5
vpxor $T, $TMP5, $TMP5
vmovdqu 5*16-32($Htbl), $TMP4
vpclmulqdq \$0x01, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
vpclmulqdq \$0x11, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP1, $TMP1
vpclmulqdq \$0x00, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP2, $TMP2
vpclmulqdq \$0x10, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
${\$aes_round_dec->(8)}
vpsrldq \$8, $TMP0, $TMP3
vpxor $TMP3, $TMP1, $TMP4
vpslldq \$8, $TMP0, $TMP3
vpxor $TMP3, $TMP2, $T
vmovdqa poly(%rip), $TMP2
${\$aes_round_dec->(9)}
___
if ($aes256) {
$code.=<<___;
${\$aes_round_dec->(10)}
${\$aes_round_dec->(11)}
${\$aes_round_dec->(12)}
${\$aes_round_dec->(13)}
vmovdqu 14*16($KS), $TMP5
___
} else {
$code.=<<___;
vmovdqu 10*16($KS), $TMP5
___
}
$code.=<<___;
vpalignr \$8, $T, $T, $TMP1
vpclmulqdq \$0x10, $TMP2, $T, $T
vpxor $T, $TMP1, $T
vpxor 0*16($CT), $TMP5, $TMP3
vaesenclast $TMP3, $CTR1, $CTR1
vpxor 1*16($CT), $TMP5, $TMP3
vaesenclast $TMP3, $CTR2, $CTR2
vpxor 2*16($CT), $TMP5, $TMP3
vaesenclast $TMP3, $CTR3, $CTR3
vpxor 3*16($CT), $TMP5, $TMP3
vaesenclast $TMP3, $CTR4, $CTR4
vpxor 4*16($CT), $TMP5, $TMP3
vaesenclast $TMP3, $CTR5, $CTR5
vpxor 5*16($CT), $TMP5, $TMP3
vaesenclast $TMP3, $CTR6, $CTR6
vpalignr \$8, $T, $T, $TMP1
vpclmulqdq \$0x10, $TMP2, $T, $T
vpxor $T, $TMP1, $T
vmovdqu $CTR1, 0*16($PT)
vmovdqu $CTR2, 1*16($PT)
vmovdqu $CTR3, 2*16($PT)
vmovdqu $CTR4, 3*16($PT)
vmovdqu $CTR5, 4*16($PT)
vmovdqu $CTR6, 5*16($PT)
vpxor $TMP4, $T, $T
lea 96($CT), $CT
lea 96($PT), $PT
jmp .L${labelPrefix}_dec_loop1
.L${labelPrefix}_dec_finish_96:
vmovdqa $CTR6, $TMP5
vmovdqa $CTR5, 1*16-32($secureBuffer)
vmovdqa $CTR4, 2*16-32($secureBuffer)
vmovdqa $CTR3, 3*16-32($secureBuffer)
vmovdqa $CTR2, 4*16-32($secureBuffer)
vmovdqa $CTR1, 5*16-32($secureBuffer)
vmovdqu 0*16-32($Htbl), $TMP3
vpclmulqdq \$0x10, $TMP3, $TMP5, $TMP0
vpclmulqdq \$0x11, $TMP3, $TMP5, $TMP1
vpclmulqdq \$0x00, $TMP3, $TMP5, $TMP2
vpclmulqdq \$0x01, $TMP3, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
${\$schoolbook->(1)}
${\$schoolbook->(2)}
${\$schoolbook->(3)}
${\$schoolbook->(4)}
vmovdqu 5*16-32($secureBuffer), $TMP5
vpxor $T, $TMP5, $TMP5
vmovdqu 5*16-32($Htbl), $TMP4
vpclmulqdq \$0x11, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP1, $TMP1
vpclmulqdq \$0x00, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP2, $TMP2
vpclmulqdq \$0x10, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
vpclmulqdq \$0x01, $TMP4, $TMP5, $TMP3
vpxor $TMP3, $TMP0, $TMP0
vpsrldq \$8, $TMP0, $TMP3
vpxor $TMP3, $TMP1, $TMP4
vpslldq \$8, $TMP0, $TMP3
vpxor $TMP3, $TMP2, $T
vmovdqa poly(%rip), $TMP2
vpalignr \$8, $T, $T, $TMP1
vpclmulqdq \$0x10, $TMP2, $T, $T
vpxor $T, $TMP1, $T
vpalignr \$8, $T, $T, $TMP1
vpclmulqdq \$0x10, $TMP2, $T, $T
vpxor $T, $TMP1, $T
vpxor $TMP4, $T, $T
.L${labelPrefix}_dec_loop2:
# Here we encrypt any remaining whole block
# if there are no whole blocks
cmp \$16, $LEN
jb .L${labelPrefix}_dec_out
sub \$16, $LEN
vmovdqa $CTR, $TMP1
vpaddd one(%rip), $CTR, $CTR
vpxor 0*16($KS), $TMP1, $TMP1
vaesenc 1*16($KS), $TMP1, $TMP1
vaesenc 2*16($KS), $TMP1, $TMP1
vaesenc 3*16($KS), $TMP1, $TMP1
vaesenc 4*16($KS), $TMP1, $TMP1
vaesenc 5*16($KS), $TMP1, $TMP1
vaesenc 6*16($KS), $TMP1, $TMP1
vaesenc 7*16($KS), $TMP1, $TMP1
vaesenc 8*16($KS), $TMP1, $TMP1
vaesenc 9*16($KS), $TMP1, $TMP1
___
if ($aes256) {
$code.=<<___;
vaesenc 10*16($KS), $TMP1, $TMP1
vaesenc 11*16($KS), $TMP1, $TMP1
vaesenc 12*16($KS), $TMP1, $TMP1
vaesenc 13*16($KS), $TMP1, $TMP1
vaesenclast 14*16($KS), $TMP1, $TMP1
___
} else {
$code.=<<___;
vaesenclast 10*16($KS), $TMP1, $TMP1
___
}
$code.=<<___;
vpxor ($CT), $TMP1, $TMP1
vmovdqu $TMP1, ($PT)
addq \$16, $CT
addq \$16, $PT
vpxor $TMP1, $T, $T
vmovdqa -32($Htbl), $TMP0
call GFMUL
jmp .L${labelPrefix}_dec_loop2
.L${labelPrefix}_dec_out:
vmovdqu $T, ($POL)
ret
.cfi_endproc
___
if ($aes256) {
$code.=<<___;
.size aes256gcmsiv_dec, .-aes256gcmsiv_dec
___
} else {
$code.=<<___;
.size aes128gcmsiv_dec, .-aes128gcmsiv_dec
___
}
}
aesgcmsiv_dec(0); # emit 128-bit version
sub aes128gcmsiv_ecb_enc_block {
my $STATE_1 = "%xmm1";
my $KSp = "%rdx";
# parameter 1: PT %rdi (pointer to 128 bit)
# parameter 2: CT %rsi (pointer to 128 bit)
# parameter 3: ks %rdx (pointer to ks)
$code.=<<___;
.globl aes128gcmsiv_ecb_enc_block
.type aes128gcmsiv_ecb_enc_block,\@function,3
.align 16
aes128gcmsiv_ecb_enc_block:
.cfi_startproc
_CET_ENDBR
vmovdqa (%rdi), $STATE_1
vpxor ($KSp), $STATE_1, $STATE_1
vaesenc 1*16($KSp), $STATE_1, $STATE_1
vaesenc 2*16($KSp), $STATE_1, $STATE_1
vaesenc 3*16($KSp), $STATE_1, $STATE_1
vaesenc 4*16($KSp), $STATE_1, $STATE_1
vaesenc 5*16($KSp), $STATE_1, $STATE_1
vaesenc 6*16($KSp), $STATE_1, $STATE_1
vaesenc 7*16($KSp), $STATE_1, $STATE_1
vaesenc 8*16($KSp), $STATE_1, $STATE_1
vaesenc 9*16($KSp), $STATE_1, $STATE_1
vaesenclast 10*16($KSp), $STATE_1, $STATE_1 # STATE_1 == IV
vmovdqa $STATE_1, (%rsi)
ret
.cfi_endproc
.size aes128gcmsiv_ecb_enc_block,.-aes128gcmsiv_ecb_enc_block
___
}
aes128gcmsiv_ecb_enc_block();
sub aes256gcmsiv_aes_ks_enc_x1 {
my $KS = "%rdx";
my $KEYp = "%rcx";
my $CON_MASK = "%xmm0";
my $MASK_256 = "%xmm15";
my $KEY_1 = "%xmm1";
my $KEY_2 = "%xmm3";
my $BLOCK1 = "%xmm8";
my $AUX_REG = "%xmm14";
my $PT = "%rdi";
my $CT = "%rsi";
my $round_double = sub {
my ($i, $j) = @_;
return <<___;
vpshufb %xmm15, %xmm3, %xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpslld \$1, %xmm0, %xmm0
vpslldq \$4, %xmm1, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpslldq \$4, %xmm4, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpslldq \$4, %xmm4, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vaesenc %xmm1, $BLOCK1, $BLOCK1
vmovdqu %xmm1, ${\eval(16*$i)}($KS)
vpshufd \$0xff, %xmm1, %xmm2
vaesenclast %xmm14, %xmm2, %xmm2
vpslldq \$4, %xmm3, %xmm4
vpxor %xmm4, %xmm3, %xmm3
vpslldq \$4, %xmm4, %xmm4
vpxor %xmm4, %xmm3, %xmm3
vpslldq \$4, %xmm4, %xmm4
vpxor %xmm4, %xmm3, %xmm3
vpxor %xmm2, %xmm3, %xmm3
vaesenc %xmm3, $BLOCK1, $BLOCK1
vmovdqu %xmm3, ${\eval(16*$j)}($KS)
___
};
my $round_last = sub {
my ($i) = @_;
return <<___;
vpshufb %xmm15, %xmm3, %xmm2
vaesenclast %xmm0, %xmm2, %xmm2
vpslldq \$4, %xmm1, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpslldq \$4, %xmm4, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpslldq \$4, %xmm4, %xmm4
vpxor %xmm4, %xmm1, %xmm1
vpxor %xmm2, %xmm1, %xmm1
vaesenclast %xmm1, $BLOCK1, $BLOCK1
vmovdqu %xmm1, ${\eval(16*$i)}($KS)
___
};
# parameter 1: %rdi Pointer to PT1
# parameter 2: %rsi Pointer to CT1
# parameter 3: %rdx Pointer to KS
# parameter 4: %rcx Pointer to initial key
$code.=<<___;
.globl aes256gcmsiv_aes_ks_enc_x1
.type aes256gcmsiv_aes_ks_enc_x1,\@function,4
.align 16
aes256gcmsiv_aes_ks_enc_x1:
.cfi_startproc
_CET_ENDBR
vmovdqa con1(%rip), $CON_MASK # CON_MASK = 1,1,1,1
vmovdqa mask(%rip), $MASK_256 # MASK_256
vmovdqa ($PT), $BLOCK1
vmovdqa ($KEYp), $KEY_1 # KEY_1 || KEY_2 [0..7] = user key
vmovdqa 16($KEYp), $KEY_2
vpxor $KEY_1, $BLOCK1, $BLOCK1
vaesenc $KEY_2, $BLOCK1, $BLOCK1
vmovdqu $KEY_1, ($KS) # First round key
vmovdqu $KEY_2, 16($KS)
vpxor $AUX_REG, $AUX_REG, $AUX_REG
${\$round_double->(2, 3)}
${\$round_double->(4, 5)}
${\$round_double->(6, 7)}
${\$round_double->(8, 9)}
${\$round_double->(10, 11)}
${\$round_double->(12, 13)}
${\$round_last->(14)}
vmovdqa $BLOCK1, ($CT)
ret
.cfi_endproc
.size aes256gcmsiv_aes_ks_enc_x1,.-aes256gcmsiv_aes_ks_enc_x1
___
}
aes256gcmsiv_aes_ks_enc_x1();
sub aes256gcmsiv_ecb_enc_block {
my $STATE_1 = "%xmm1";
my $PT = "%rdi";
my $CT = "%rsi";
my $KSp = "%rdx";
# parameter 1: PT %rdi (pointer to 128 bit)
# parameter 2: CT %rsi (pointer to 128 bit)
# parameter 3: ks %rdx (pointer to ks)
$code.=<<___;
.globl aes256gcmsiv_ecb_enc_block
.type aes256gcmsiv_ecb_enc_block,\@function,3
.align 16
aes256gcmsiv_ecb_enc_block:
.cfi_startproc
_CET_ENDBR
vmovdqa (%rdi), $STATE_1
vpxor ($KSp), $STATE_1, $STATE_1
vaesenc 1*16($KSp), $STATE_1, $STATE_1
vaesenc 2*16($KSp), $STATE_1, $STATE_1
vaesenc 3*16($KSp), $STATE_1, $STATE_1
vaesenc 4*16($KSp), $STATE_1, $STATE_1
vaesenc 5*16($KSp), $STATE_1, $STATE_1
vaesenc 6*16($KSp), $STATE_1, $STATE_1
vaesenc 7*16($KSp), $STATE_1, $STATE_1
vaesenc 8*16($KSp), $STATE_1, $STATE_1
vaesenc 9*16($KSp), $STATE_1, $STATE_1
vaesenc 10*16($KSp), $STATE_1, $STATE_1
vaesenc 11*16($KSp), $STATE_1, $STATE_1
vaesenc 12*16($KSp), $STATE_1, $STATE_1
vaesenc 13*16($KSp), $STATE_1, $STATE_1
vaesenclast 14*16($KSp), $STATE_1, $STATE_1 # $STATE_1 == IV
vmovdqa $STATE_1, (%rsi)
ret
.cfi_endproc
.size aes256gcmsiv_ecb_enc_block,.-aes256gcmsiv_ecb_enc_block
___
}
aes256gcmsiv_ecb_enc_block();
sub aes256gcmsiv_enc_msg_x4 {
my $CTR1 = "%xmm0";
my $CTR2 = "%xmm1";
my $CTR3 = "%xmm2";
my $CTR4 = "%xmm3";
my $ADDER = "%xmm4";
my $STATE1 = "%xmm5";
my $STATE2 = "%xmm6";
my $STATE3 = "%xmm7";
my $STATE4 = "%xmm8";
my $TMP = "%xmm12";
my $TMP2 = "%xmm13";
my $TMP3 = "%xmm14";
my $IV = "%xmm15";
my $PT = "%rdi";
my $CT = "%rsi";
my $TAG = "%rdx";
my $KS = "%rcx";
my $LEN = "%r8";
my $aes_round = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $TMP
vaesenc $TMP, $STATE1, $STATE1
vaesenc $TMP, $STATE2, $STATE2
vaesenc $TMP, $STATE3, $STATE3
vaesenc $TMP, $STATE4, $STATE4
___
};
my $aes_lastround = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $TMP
vaesenclast $TMP, $STATE1, $STATE1
vaesenclast $TMP, $STATE2, $STATE2
vaesenclast $TMP, $STATE3, $STATE3
vaesenclast $TMP, $STATE4, $STATE4
___
};
# void aes256gcmsiv_enc_msg_x4(unsigned char* PT, unsigned char* CT,
# unsigned char* TAG, unsigned char* KS,
# size_t byte_len);
# parameter 1: %rdi #PT
# parameter 2: %rsi #CT
# parameter 3: %rdx #TAG [127 126 ... 0] IV=[127...32]
# parameter 4: %rcx #KS
# parameter 5: %r8 #LEN MSG_length in bytes
$code.=<<___;
.globl aes256gcmsiv_enc_msg_x4
.type aes256gcmsiv_enc_msg_x4,\@function,5
.align 16
aes256gcmsiv_enc_msg_x4:
.cfi_startproc
_CET_ENDBR
test $LEN, $LEN
jnz .L256_enc_msg_x4_start
ret
.L256_enc_msg_x4_start:
movq $LEN, %r10
shrq \$4, $LEN # LEN = num of blocks
shlq \$60, %r10
jz .L256_enc_msg_x4_start2
addq \$1, $LEN
.L256_enc_msg_x4_start2:
movq $LEN, %r10
shlq \$62, %r10
shrq \$62, %r10
# make IV from TAG
vmovdqa ($TAG), $IV
vpor OR_MASK(%rip), $IV, $IV # IV = [1]TAG[126...32][00..00]
vmovdqa four(%rip), $ADDER # Register to increment counters
vmovdqa $IV, $CTR1 # CTR1 = TAG[1][127...32][00..00]
vpaddd one(%rip), $IV, $CTR2 # CTR2 = TAG[1][127...32][00..01]
vpaddd two(%rip), $IV, $CTR3 # CTR3 = TAG[1][127...32][00..02]
vpaddd three(%rip), $IV, $CTR4 # CTR4 = TAG[1][127...32][00..03]
shrq \$2, $LEN
je .L256_enc_msg_x4_check_remainder
subq \$64, $CT
subq \$64, $PT
.L256_enc_msg_x4_loop1:
addq \$64, $CT
addq \$64, $PT
vmovdqa $CTR1, $STATE1
vmovdqa $CTR2, $STATE2
vmovdqa $CTR3, $STATE3
vmovdqa $CTR4, $STATE4
vpxor ($KS), $STATE1, $STATE1
vpxor ($KS), $STATE2, $STATE2
vpxor ($KS), $STATE3, $STATE3
vpxor ($KS), $STATE4, $STATE4
${\$aes_round->(1)}
vpaddd $ADDER, $CTR1, $CTR1
${\$aes_round->(2)}
vpaddd $ADDER, $CTR2, $CTR2
${\$aes_round->(3)}
vpaddd $ADDER, $CTR3, $CTR3
${\$aes_round->(4)}
vpaddd $ADDER, $CTR4, $CTR4
${\$aes_round->(5)}
${\$aes_round->(6)}
${\$aes_round->(7)}
${\$aes_round->(8)}
${\$aes_round->(9)}
${\$aes_round->(10)}
${\$aes_round->(11)}
${\$aes_round->(12)}
${\$aes_round->(13)}
${\$aes_lastround->(14)}
# XOR with Plaintext
vpxor 0*16($PT), $STATE1, $STATE1
vpxor 1*16($PT), $STATE2, $STATE2
vpxor 2*16($PT), $STATE3, $STATE3
vpxor 3*16($PT), $STATE4, $STATE4
subq \$1, $LEN
vmovdqu $STATE1, 0*16($CT)
vmovdqu $STATE2, 1*16($CT)
vmovdqu $STATE3, 2*16($CT)
vmovdqu $STATE4, 3*16($CT)
jne .L256_enc_msg_x4_loop1
addq \$64, $CT
addq \$64, $PT
.L256_enc_msg_x4_check_remainder:
cmpq \$0, %r10
je .L256_enc_msg_x4_out
.L256_enc_msg_x4_loop2:
# encrypt each block separately
# CTR1 is the highest counter (even if no LOOP done)
vmovdqa $CTR1, $STATE1
vpaddd one(%rip), $CTR1, $CTR1 # inc counter
vpxor ($KS), $STATE1, $STATE1
vaesenc 16($KS), $STATE1, $STATE1
vaesenc 32($KS), $STATE1, $STATE1
vaesenc 48($KS), $STATE1, $STATE1
vaesenc 64($KS), $STATE1, $STATE1
vaesenc 80($KS), $STATE1, $STATE1
vaesenc 96($KS), $STATE1, $STATE1
vaesenc 112($KS), $STATE1, $STATE1
vaesenc 128($KS), $STATE1, $STATE1
vaesenc 144($KS), $STATE1, $STATE1
vaesenc 160($KS), $STATE1, $STATE1
vaesenc 176($KS), $STATE1, $STATE1
vaesenc 192($KS), $STATE1, $STATE1
vaesenc 208($KS), $STATE1, $STATE1
vaesenclast 224($KS), $STATE1, $STATE1
# XOR with Plaintext
vpxor ($PT), $STATE1, $STATE1
vmovdqu $STATE1, ($CT)
addq \$16, $PT
addq \$16, $CT
subq \$1, %r10
jne .L256_enc_msg_x4_loop2
.L256_enc_msg_x4_out:
ret
.cfi_endproc
.size aes256gcmsiv_enc_msg_x4,.-aes256gcmsiv_enc_msg_x4
___
}
aes256gcmsiv_enc_msg_x4();
sub aes256gcmsiv_enc_msg_x8() {
my $STATE1 = "%xmm1";
my $STATE2 = "%xmm2";
my $STATE3 = "%xmm3";
my $STATE4 = "%xmm4";
my $STATE5 = "%xmm5";
my $STATE6 = "%xmm6";
my $STATE7 = "%xmm7";
my $STATE8 = "%xmm8";
my $CTR1 = "%xmm0";
my $CTR2 = "%xmm9";
my $CTR3 = "%xmm10";
my $CTR4 = "%xmm11";
my $CTR5 = "%xmm12";
my $CTR6 = "%xmm13";
my $CTR7 = "%xmm14";
my $TMP1 = "%xmm1";
my $TMP2 = "%xmm2";
my $KS = "%rcx";
my $LEN = "%r8";
my $PT = "%rdi";
my $CT = "%rsi";
my $TAG = "%rdx";
my $SCHED = "%xmm15";
my $aes_round8 = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $SCHED
vaesenc $SCHED, $STATE1, $STATE1
vaesenc $SCHED, $STATE2, $STATE2
vaesenc $SCHED, $STATE3, $STATE3
vaesenc $SCHED, $STATE4, $STATE4
vaesenc $SCHED, $STATE5, $STATE5
vaesenc $SCHED, $STATE6, $STATE6
vaesenc $SCHED, $STATE7, $STATE7
vaesenc $SCHED, $STATE8, $STATE8
___
};
my $aes_lastround8 = sub {
my ($i) = @_;
return <<___;
vmovdqu ${\eval($i*16)}($KS), $SCHED
vaesenclast $SCHED, $STATE1, $STATE1
vaesenclast $SCHED, $STATE2, $STATE2
vaesenclast $SCHED, $STATE3, $STATE3
vaesenclast $SCHED, $STATE4, $STATE4
vaesenclast $SCHED, $STATE5, $STATE5
vaesenclast $SCHED, $STATE6, $STATE6
vaesenclast $SCHED, $STATE7, $STATE7
vaesenclast $SCHED, $STATE8, $STATE8
___
};
# void ENC_MSG_x8(unsigned char* PT,
# unsigned char* CT,
# unsigned char* TAG,
# unsigned char* KS,
# size_t byte_len);
# parameter 1: %rdi #PT
# parameter 2: %rsi #CT
# parameter 3: %rdx #TAG [127 126 ... 0] IV=[127...32]
# parameter 4: %rcx #KS
# parameter 5: %r8 #LEN MSG_length in bytes
$code.=<<___;
.globl aes256gcmsiv_enc_msg_x8
.type aes256gcmsiv_enc_msg_x8,\@function,5
.align 16
aes256gcmsiv_enc_msg_x8:
.cfi_startproc
_CET_ENDBR
test $LEN, $LEN
jnz .L256_enc_msg_x8_start
ret
.L256_enc_msg_x8_start:
# Place in stack
movq %rsp, %r11
subq \$16, %r11
andq \$-64, %r11
movq $LEN, %r10
shrq \$4, $LEN # LEN = num of blocks
shlq \$60, %r10
jz .L256_enc_msg_x8_start2
addq \$1, $LEN
.L256_enc_msg_x8_start2:
movq $LEN, %r10
shlq \$61, %r10
shrq \$61, %r10
# Make IV from TAG
vmovdqa ($TAG), $TMP1
vpor OR_MASK(%rip), $TMP1, $TMP1 # TMP1= IV = [1]TAG[126...32][00..00]
# store counter8 on the stack
vpaddd seven(%rip), $TMP1, $CTR1
vmovdqa $CTR1, (%r11) # CTR8 = TAG[127...32][00..07]
vpaddd one(%rip), $TMP1, $CTR2 # CTR2 = TAG[127...32][00..01]
vpaddd two(%rip), $TMP1, $CTR3 # CTR3 = TAG[127...32][00..02]
vpaddd three(%rip), $TMP1, $CTR4 # CTR4 = TAG[127...32][00..03]
vpaddd four(%rip), $TMP1, $CTR5 # CTR5 = TAG[127...32][00..04]
vpaddd five(%rip), $TMP1, $CTR6 # CTR6 = TAG[127...32][00..05]
vpaddd six(%rip), $TMP1, $CTR7 # CTR7 = TAG[127...32][00..06]
vmovdqa $TMP1, $CTR1 # CTR1 = TAG[127...32][00..00]
shrq \$3, $LEN
jz .L256_enc_msg_x8_check_remainder
subq \$128, $CT
subq \$128, $PT
.L256_enc_msg_x8_loop1:
addq \$128, $CT
addq \$128, $PT
vmovdqa $CTR1, $STATE1
vmovdqa $CTR2, $STATE2
vmovdqa $CTR3, $STATE3
vmovdqa $CTR4, $STATE4
vmovdqa $CTR5, $STATE5
vmovdqa $CTR6, $STATE6
vmovdqa $CTR7, $STATE7
# move from stack
vmovdqa (%r11), $STATE8
vpxor ($KS), $STATE1, $STATE1
vpxor ($KS), $STATE2, $STATE2
vpxor ($KS), $STATE3, $STATE3
vpxor ($KS), $STATE4, $STATE4
vpxor ($KS), $STATE5, $STATE5
vpxor ($KS), $STATE6, $STATE6
vpxor ($KS), $STATE7, $STATE7
vpxor ($KS), $STATE8, $STATE8
${\$aes_round8->(1)}
vmovdqa (%r11), $CTR7 # deal with CTR8
vpaddd eight(%rip), $CTR7, $CTR7
vmovdqa $CTR7, (%r11)
${\$aes_round8->(2)}
vpsubd one(%rip), $CTR7, $CTR7
${\$aes_round8->(3)}
vpaddd eight(%rip), $CTR1, $CTR1
${\$aes_round8->(4)}
vpaddd eight(%rip), $CTR2, $CTR2
${\$aes_round8->(5)}
vpaddd eight(%rip), $CTR3, $CTR3
${\$aes_round8->(6)}
vpaddd eight(%rip), $CTR4, $CTR4
${\$aes_round8->(7)}
vpaddd eight(%rip), $CTR5, $CTR5
${\$aes_round8->(8)}
vpaddd eight(%rip), $CTR6, $CTR6
${\$aes_round8->(9)}
${\$aes_round8->(10)}
${\$aes_round8->(11)}
${\$aes_round8->(12)}
${\$aes_round8->(13)}
${\$aes_lastround8->(14)}
# XOR with Plaintext
vpxor 0*16($PT), $STATE1, $STATE1
vpxor 1*16($PT), $STATE2, $STATE2
vpxor 2*16($PT), $STATE3, $STATE3
vpxor 3*16($PT), $STATE4, $STATE4
vpxor 4*16($PT), $STATE5, $STATE5
vpxor 5*16($PT), $STATE6, $STATE6
vpxor 6*16($PT), $STATE7, $STATE7
vpxor 7*16($PT), $STATE8, $STATE8
subq \$1, $LEN
vmovdqu $STATE1, 0*16($CT)
vmovdqu $STATE2, 1*16($CT)
vmovdqu $STATE3, 2*16($CT)
vmovdqu $STATE4, 3*16($CT)
vmovdqu $STATE5, 4*16($CT)
vmovdqu $STATE6, 5*16($CT)
vmovdqu $STATE7, 6*16($CT)
vmovdqu $STATE8, 7*16($CT)
jne .L256_enc_msg_x8_loop1
addq \$128, $CT
addq \$128, $PT
.L256_enc_msg_x8_check_remainder:
cmpq \$0, %r10
je .L256_enc_msg_x8_out
.L256_enc_msg_x8_loop2:
# encrypt each block separately
# CTR1 is the highest counter (even if no LOOP done)
vmovdqa $CTR1, $STATE1
vpaddd one(%rip), $CTR1, $CTR1
vpxor ($KS), $STATE1, $STATE1
vaesenc 16($KS), $STATE1, $STATE1
vaesenc 32($KS), $STATE1, $STATE1
vaesenc 48($KS), $STATE1, $STATE1
vaesenc 64($KS), $STATE1, $STATE1
vaesenc 80($KS), $STATE1, $STATE1
vaesenc 96($KS), $STATE1, $STATE1
vaesenc 112($KS), $STATE1, $STATE1
vaesenc 128($KS), $STATE1, $STATE1
vaesenc 144($KS), $STATE1, $STATE1
vaesenc 160($KS), $STATE1, $STATE1
vaesenc 176($KS), $STATE1, $STATE1
vaesenc 192($KS), $STATE1, $STATE1
vaesenc 208($KS), $STATE1, $STATE1
vaesenclast 224($KS), $STATE1, $STATE1
# XOR with Plaintext
vpxor ($PT), $STATE1, $STATE1
vmovdqu $STATE1, ($CT)
addq \$16, $PT
addq \$16, $CT
subq \$1, %r10
jnz .L256_enc_msg_x8_loop2
.L256_enc_msg_x8_out:
ret
.cfi_endproc
.size aes256gcmsiv_enc_msg_x8,.-aes256gcmsiv_enc_msg_x8
___
}
aes256gcmsiv_enc_msg_x8();
aesgcmsiv_dec(1);
sub aes256gcmsiv_kdf {
my $ONE = "%xmm8";
my $BLOCK1 = "%xmm4";
my $BLOCK2 = "%xmm6";
my $BLOCK3 = "%xmm7";
my $BLOCK4 = "%xmm11";
my $BLOCK5 = "%xmm12";
my $BLOCK6 = "%xmm13";
my $enc_roundx6 = sub {
my ($i, $j) = @_;
return <<___;
vmovdqa ${\eval($i*16)}(%rdx), $j
vaesenc $j, $BLOCK1, $BLOCK1
vaesenc $j, $BLOCK2, $BLOCK2
vaesenc $j, $BLOCK3, $BLOCK3
vaesenc $j, $BLOCK4, $BLOCK4
vaesenc $j, $BLOCK5, $BLOCK5
vaesenc $j, $BLOCK6, $BLOCK6
___
};
my $enc_roundlastx6 = sub {
my ($i, $j) = @_;
return <<___;
vmovdqa ${\eval($i*16)}(%rdx), $j
vaesenclast $j, $BLOCK1, $BLOCK1
vaesenclast $j, $BLOCK2, $BLOCK2
vaesenclast $j, $BLOCK3, $BLOCK3
vaesenclast $j, $BLOCK4, $BLOCK4
vaesenclast $j, $BLOCK5, $BLOCK5
vaesenclast $j, $BLOCK6, $BLOCK6
___
};
# void aes256gcmsiv_kdf(const uint8_t nonce[16],
# uint8_t *out_key_material,
# const uint8_t *key_schedule);
$code.=<<___;
.globl aes256gcmsiv_kdf
.type aes256gcmsiv_kdf,\@function,3
.align 16
aes256gcmsiv_kdf:
.cfi_startproc
_CET_ENDBR
# parameter 1: %rdi Pointer to NONCE
# parameter 2: %rsi Pointer to CT
# parameter 4: %rdx Pointer to keys
vmovdqa (%rdx), %xmm1 # xmm1 = first 16 bytes of random key
vmovdqa 0*16(%rdi), $BLOCK1
vmovdqa and_mask(%rip), $BLOCK4
vmovdqa one(%rip), $ONE
vpshufd \$0x90, $BLOCK1, $BLOCK1
vpand $BLOCK4, $BLOCK1, $BLOCK1
vpaddd $ONE, $BLOCK1, $BLOCK2
vpaddd $ONE, $BLOCK2, $BLOCK3
vpaddd $ONE, $BLOCK3, $BLOCK4
vpaddd $ONE, $BLOCK4, $BLOCK5
vpaddd $ONE, $BLOCK5, $BLOCK6
vpxor %xmm1, $BLOCK1, $BLOCK1
vpxor %xmm1, $BLOCK2, $BLOCK2
vpxor %xmm1, $BLOCK3, $BLOCK3
vpxor %xmm1, $BLOCK4, $BLOCK4
vpxor %xmm1, $BLOCK5, $BLOCK5
vpxor %xmm1, $BLOCK6, $BLOCK6
${\$enc_roundx6->(1, "%xmm1")}
${\$enc_roundx6->(2, "%xmm2")}
${\$enc_roundx6->(3, "%xmm1")}
${\$enc_roundx6->(4, "%xmm2")}
${\$enc_roundx6->(5, "%xmm1")}
${\$enc_roundx6->(6, "%xmm2")}
${\$enc_roundx6->(7, "%xmm1")}
${\$enc_roundx6->(8, "%xmm2")}
${\$enc_roundx6->(9, "%xmm1")}
${\$enc_roundx6->(10, "%xmm2")}
${\$enc_roundx6->(11, "%xmm1")}
${\$enc_roundx6->(12, "%xmm2")}
${\$enc_roundx6->(13, "%xmm1")}
${\$enc_roundlastx6->(14, "%xmm2")}
vmovdqa $BLOCK1, 0*16(%rsi)
vmovdqa $BLOCK2, 1*16(%rsi)
vmovdqa $BLOCK3, 2*16(%rsi)
vmovdqa $BLOCK4, 3*16(%rsi)
vmovdqa $BLOCK5, 4*16(%rsi)
vmovdqa $BLOCK6, 5*16(%rsi)
ret
.cfi_endproc
.size aes256gcmsiv_kdf, .-aes256gcmsiv_kdf
___
}
aes256gcmsiv_kdf();
print $code;
close STDOUT or die "error closing STDOUT: $!";