aboutsummaryrefslogtreecommitdiff
path: root/arch/x86/crypto/aesni-intel_asm.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/crypto/aesni-intel_asm.S')
-rw-r--r--arch/x86/crypto/aesni-intel_asm.S156
1 files changed, 155 insertions, 1 deletions
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 3470624d783..477e9d75149 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -34,6 +34,10 @@
#ifdef __x86_64__
.data
+.align 16
+.Lgf128mul_x_ble_mask:
+ .octa 0x00000000000000010000000000000087
+
POLY: .octa 0xC2000000000000000000000000000001
TWOONE: .octa 0x00000001000000000000000000000001
@@ -105,6 +109,8 @@ enc: .octa 0x2
#define CTR %xmm11
#define INC %xmm12
+#define GF128MUL_MASK %xmm10
+
#ifdef __x86_64__
#define AREG %rax
#define KEYP %rdi
@@ -1262,7 +1268,6 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
* poly = x^128 + x^127 + x^126 + x^121 + 1
*
*****************************************************************************/
-
ENTRY(aesni_gcm_dec)
push %r12
push %r13
@@ -1437,6 +1442,7 @@ _return_T_done_decrypt:
pop %r13
pop %r12
ret
+ENDPROC(aesni_gcm_dec)
/*****************************************************************************
@@ -1700,10 +1706,12 @@ _return_T_done_encrypt:
pop %r13
pop %r12
ret
+ENDPROC(aesni_gcm_enc)
#endif
+.align 4
_key_expansion_128:
_key_expansion_256a:
pshufd $0b11111111, %xmm1, %xmm1
@@ -1715,6 +1723,8 @@ _key_expansion_256a:
movaps %xmm0, (TKEYP)
add $0x10, TKEYP
ret
+ENDPROC(_key_expansion_128)
+ENDPROC(_key_expansion_256a)
.align 4
_key_expansion_192a:
@@ -1739,6 +1749,7 @@ _key_expansion_192a:
movaps %xmm1, 0x10(TKEYP)
add $0x20, TKEYP
ret
+ENDPROC(_key_expansion_192a)
.align 4
_key_expansion_192b:
@@ -1758,6 +1769,7 @@ _key_expansion_192b:
movaps %xmm0, (TKEYP)
add $0x10, TKEYP
ret
+ENDPROC(_key_expansion_192b)
.align 4
_key_expansion_256b:
@@ -1770,6 +1782,7 @@ _key_expansion_256b:
movaps %xmm2, (TKEYP)
add $0x10, TKEYP
ret
+ENDPROC(_key_expansion_256b)
/*
* int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
@@ -1882,6 +1895,7 @@ ENTRY(aesni_set_key)
popl KEYP
#endif
ret
+ENDPROC(aesni_set_key)
/*
* void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
@@ -1903,6 +1917,7 @@ ENTRY(aesni_enc)
popl KEYP
#endif
ret
+ENDPROC(aesni_enc)
/*
* _aesni_enc1: internal ABI
@@ -1960,6 +1975,7 @@ _aesni_enc1:
movaps 0x70(TKEYP), KEY
AESENCLAST KEY STATE
ret
+ENDPROC(_aesni_enc1)
/*
* _aesni_enc4: internal ABI
@@ -2068,6 +2084,7 @@ _aesni_enc4:
AESENCLAST KEY STATE3
AESENCLAST KEY STATE4
ret
+ENDPROC(_aesni_enc4)
/*
* void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
@@ -2090,6 +2107,7 @@ ENTRY(aesni_dec)
popl KEYP
#endif
ret
+ENDPROC(aesni_dec)
/*
* _aesni_dec1: internal ABI
@@ -2147,6 +2165,7 @@ _aesni_dec1:
movaps 0x70(TKEYP), KEY
AESDECLAST KEY STATE
ret
+ENDPROC(_aesni_dec1)
/*
* _aesni_dec4: internal ABI
@@ -2255,6 +2274,7 @@ _aesni_dec4:
AESDECLAST KEY STATE3
AESDECLAST KEY STATE4
ret
+ENDPROC(_aesni_dec4)
/*
* void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2312,6 +2332,7 @@ ENTRY(aesni_ecb_enc)
popl LEN
#endif
ret
+ENDPROC(aesni_ecb_enc)
/*
* void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2370,6 +2391,7 @@ ENTRY(aesni_ecb_dec)
popl LEN
#endif
ret
+ENDPROC(aesni_ecb_dec)
/*
* void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2411,6 +2433,7 @@ ENTRY(aesni_cbc_enc)
popl IVP
#endif
ret
+ENDPROC(aesni_cbc_enc)
/*
* void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2501,6 +2524,7 @@ ENTRY(aesni_cbc_dec)
popl IVP
#endif
ret
+ENDPROC(aesni_cbc_dec)
#ifdef __x86_64__
.align 16
@@ -2527,6 +2551,7 @@ _aesni_inc_init:
MOVQ_R64_XMM TCTR_LOW INC
MOVQ_R64_XMM CTR TCTR_LOW
ret
+ENDPROC(_aesni_inc_init)
/*
* _aesni_inc: internal ABI
@@ -2555,6 +2580,7 @@ _aesni_inc:
movaps CTR, IV
PSHUFB_XMM BSWAP_MASK IV
ret
+ENDPROC(_aesni_inc)
/*
* void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
@@ -2615,4 +2641,132 @@ ENTRY(aesni_ctr_enc)
movups IV, (IVP)
.Lctr_enc_just_ret:
ret
+ENDPROC(aesni_ctr_enc)
+
+/*
+ * _aesni_gf128mul_x_ble: internal ABI
+ * Multiply in GF(2^128) for XTS IVs
+ * input:
+ * IV: current IV
+ * GF128MUL_MASK == mask with 0x87 and 0x01
+ * output:
+ * IV: next IV
+ * changed:
+ * CTR: == temporary value
+ */
+#define _aesni_gf128mul_x_ble() \
+ pshufd $0x13, IV, CTR; \
+ paddq IV, IV; \
+ psrad $31, CTR; \
+ pand GF128MUL_MASK, CTR; \
+ pxor CTR, IV;
+
+/*
+ * void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
+ * bool enc, u8 *iv)
+ */
+ENTRY(aesni_xts_crypt8)
+ cmpb $0, %cl
+ movl $0, %ecx
+ movl $240, %r10d
+ leaq _aesni_enc4, %r11
+ leaq _aesni_dec4, %rax
+ cmovel %r10d, %ecx
+ cmoveq %rax, %r11
+
+ movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK
+ movups (IVP), IV
+
+ mov 480(KEYP), KLEN
+ addq %rcx, KEYP
+
+ movdqa IV, STATE1
+ movdqu 0x00(INP), INC
+ pxor INC, STATE1
+ movdqu IV, 0x00(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE2
+ movdqu 0x10(INP), INC
+ pxor INC, STATE2
+ movdqu IV, 0x10(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE3
+ movdqu 0x20(INP), INC
+ pxor INC, STATE3
+ movdqu IV, 0x20(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE4
+ movdqu 0x30(INP), INC
+ pxor INC, STATE4
+ movdqu IV, 0x30(OUTP)
+
+ call *%r11
+
+ movdqu 0x00(OUTP), INC
+ pxor INC, STATE1
+ movdqu STATE1, 0x00(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE1
+ movdqu 0x40(INP), INC
+ pxor INC, STATE1
+ movdqu IV, 0x40(OUTP)
+
+ movdqu 0x10(OUTP), INC
+ pxor INC, STATE2
+ movdqu STATE2, 0x10(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE2
+ movdqu 0x50(INP), INC
+ pxor INC, STATE2
+ movdqu IV, 0x50(OUTP)
+
+ movdqu 0x20(OUTP), INC
+ pxor INC, STATE3
+ movdqu STATE3, 0x20(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE3
+ movdqu 0x60(INP), INC
+ pxor INC, STATE3
+ movdqu IV, 0x60(OUTP)
+
+ movdqu 0x30(OUTP), INC
+ pxor INC, STATE4
+ movdqu STATE4, 0x30(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movdqa IV, STATE4
+ movdqu 0x70(INP), INC
+ pxor INC, STATE4
+ movdqu IV, 0x70(OUTP)
+
+ _aesni_gf128mul_x_ble()
+ movups IV, (IVP)
+
+ call *%r11
+
+ movdqu 0x40(OUTP), INC
+ pxor INC, STATE1
+ movdqu STATE1, 0x40(OUTP)
+
+ movdqu 0x50(OUTP), INC
+ pxor INC, STATE2
+ movdqu STATE2, 0x50(OUTP)
+
+ movdqu 0x60(OUTP), INC
+ pxor INC, STATE3
+ movdqu STATE3, 0x60(OUTP)
+
+ movdqu 0x70(OUTP), INC
+ pxor INC, STATE4
+ movdqu STATE4, 0x70(OUTP)
+
+ ret
+ENDPROC(aesni_xts_crypt8)
+
#endif
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-10 18:55:59 +0000