// Copyright (c) 2022, ARM Inc. // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. // Permission to use, copy, modify, and/or distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ // Author: Hanno Becker // This file was derived from the assembly generated from aesv8-gcm-armv8.pl, // written by Fangming Fang for the OpenSSL project, // and derived from https: // github.com/ARM-software/AArch64cryptolib, original // author Samuel Lee . // The code below is a 'clean' AArch64 implementation of AES-GCM emphasizing // the logic of the computation. It is meant as the input to manual audits / // formal verification, as well as automated micro-optimization such as done // by the SLOTHY superoptimizer (https: // github.com/slothy-optimizer/slothy). #if !defined(__has_feature) #define __has_feature(x) 0 #endif #if __has_feature(memory_sanitizer) && !defined(OPENSSL_NO_ASM) #define OPENSSL_NO_ASM #endif #include "aarch64-gcm-asm-compat.h" #if !defined(OPENSSL_NO_ASM) && defined(__AARCH64EL__) #if defined(__ELF__) .arch armv8-a+crypto .text .globl aes_gcm_enc_kernel_slothy_base_128 .hidden aes_gcm_enc_kernel_slothy_base_128 .type aes_gcm_enc_kernel_slothy_base_128,%function #elif defined(__APPLE__) .text .globl _aes_gcm_enc_kernel_slothy_base_128 .private_extern _aes_gcm_enc_kernel_slothy_base_128 #else #error Unknown configuration #endif #if __ARM_MAX_ARCH__ >= 8 // Arguments input .req x0 len_bits .req x1 output .req x2 tag_ptr .req x3 ivec .req x4 key .req x5 Htable .req x6 byte_len .req x15 constant_temp .req x25 count .req x1 full_blocks .req x7 remainder .req x16 unroll .req x10 ctr_tmp .req x14 ctr_tmp_w .req w14 ivec_0_63 .req x11 ivec_64_96 .req x12 ivec_64_96_w .req w12 ctr .req w13 ctr_x .req x13 aes_st .req v0 aes_st_q .req q0 aes_st_d .req d0 res .req v0 res_q .req q0 ghash_hi .req v9 ghash_lo .req v8 ghash_mid .req v10 ghash_mid_d .req d10 ghash_tmp .req v11 ghash_tmp_d .req d11 ghash_mod .req v7 ghash_mod_d .req d7 modulo_tmp0 .req v0 modulo_tmp1 .req v1 Ht1q .req q12 Ht2q .req q13 Ht12q .req q14 Ht1 .req v12 Ht2 .req v13 Ht12 .req v14 Ht3q .req q15 Ht4q .req q16 Ht34q .req q17 Ht3 .req v15 Ht4 .req v16 Ht34 .req v17 rk0q .req q18 rk1q .req q19 rk2q .req q20 rk3q .req q21 rk4q .req q22 rk5q .req q23 rk6q .req q24 rk7q .req q25 rk8q .req q26 rk9q .req q27 rk10q .req q28 rk0 .req v18 rk1 .req v19 rk2 .req v20 rk3 .req v21 rk4 .req v22 rk5 .req v23 rk6 .req v24 rk7 .req v25 rk8 .req v26 rk9 .req v27 rk10 .req v28 plain .req v29 plain_q .req q29 tag .req v30 tag_q .req q30 #define UNROLL 4 #define STACK_SIZE_GPRS (6*16) #define STACK_SIZE_VREGS (4*16) #define STACK_SIZE (STACK_SIZE_GPRS + STACK_SIZE_VREGS + UNROLL*16) #define STACK_BASE_GPRS (0) #define STACK_BASE_VREGS (STACK_SIZE_GPRS) #define STACK_BASE_AES_ST (STACK_SIZE_GPRS + STACK_SIZE_VREGS) /********************************************************************/ /* Generic preamble/postamble macros */ /********************************************************************/ .macro save_vregs stp d8, d9, [sp, #(STACK_BASE_VREGS + 16*0)] stp d10, d11, [sp, #(STACK_BASE_VREGS + 16*1)] stp d12, d13, [sp, #(STACK_BASE_VREGS + 16*2)] stp d14, d15, [sp, #(STACK_BASE_VREGS + 16*3)] .endm .macro restore_vregs ldp d8, d9, [sp, #(STACK_BASE_VREGS + 16*0)] ldp d10, d11, [sp, #(STACK_BASE_VREGS + 16*1)] ldp d12, d13, [sp, #(STACK_BASE_VREGS + 16*2)] ldp d14, d15, [sp, #(STACK_BASE_VREGS + 16*3)] .endm .macro save_gprs stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] .endm .macro restore_gprs ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] .endm // Derive number of iterations of unrolled loop and single-block loop .macro prepare_loop_counts mov unroll, #UNROLL // Number of AES Blocks (16b each) lsr full_blocks, byte_len, #4 // Number of iterations of the unrolled loop udiv count, full_blocks, unroll // Number of iterations for the tail loop handling 1 block each msub remainder, count, unroll, full_blocks .endm /********************************************************************/ /* AES related macros */ /********************************************************************/ .macro load_iv ldp ivec_0_63, ivec_64_96, [ivec] lsr ctr_x, ivec_64_96, #32 rev ctr, ctr orr ivec_64_96_w, ivec_64_96_w, ivec_64_96_w // clear top 32 bit .endm .macro next_ctr_init_aes aes_st, loc add ctr_tmp_w, ctr, #\loc rev ctr_tmp_w, ctr_tmp_w orr ctr_tmp, ivec_64_96, ctr_tmp, lsl #32 stp ivec_0_63, ctr_tmp, [sp, #(STACK_BASE_AES_ST + \loc*16)] // @slothy:writes=stack_\loc ldr \aes_st\()_q, [sp, #(STACK_BASE_AES_ST + \loc*16)] // @slothy:reads=stack_\loc .endm // A single AES round // Prevent SLOTHY from unfolding because uArchs tend to fuse AESMC+AESE .macro aesr data, key // @slothy:no-unfold aese \data, \key aesmc \data, \data .endm .macro aesr_0_8 data, key aesr \data\().16b, \key\()0.16b aesr \data\().16b, \key\()1.16b aesr \data\().16b, \key\()2.16b aesr \data\().16b, \key\()3.16b aesr \data\().16b, \key\()4.16b aesr \data\().16b, \key\()5.16b aesr \data\().16b, \key\()6.16b aesr \data\().16b, \key\()7.16b aesr \data\().16b, \key\()8.16b .endm .macro aesr_9_10 data, key aesr \data\().16b, \key\()9.16b aesr \data\().16b, \key\()10.16b .endm .macro aesr_11_12 data, key aesr \data\().16b, \key\()11.16b aesr \data\().16b, \key\()12.16b .endm // Destructs inA .macro eor3 out, inA, inB, inC eor \inA, \inA, \inB eor \out, \inA, \inC .endm .macro aesr_final aes_st, plain, out aese \aes_st\().16b, rk9.16b eor3 \out\().16b, \plain\().16b, rk10.16b, \aes_st\().16b .endm .macro aes_full_block aes_st, input, output, loc next_ctr_init_aes \aes_st, \loc aesr_0_8 \aes_st\(), rk aesr_final \aes_st, \input, \output .endm .macro load_round_key i ldr rk\i\()q, [key, #((\i)*16)] .endm .macro load_round_keys load_round_key 0 load_round_key 1 load_round_key 2 load_round_key 3 load_round_key 4 load_round_key 5 load_round_key 6 load_round_key 7 load_round_key 8 load_round_key 9 load_round_key 10 .endm /********************************************************************/ /* Loading of H-table (precomputed H-powers for GHASH) */ /********************************************************************/ // This has to be synchronized with the H-table generation .macro load_h1 dst, dst_q ldr \dst_q, [Htable] .endm .macro load_h2 dst, dst_q ldr \dst_q, [Htable, #32] .endm .macro load_h3 dst, dst_q ldr \dst_q, [Htable, #48] .endm .macro load_h4 dst, dst_q ldr \dst_q, [Htable, #80] .endm .macro load_h5 dst, dst_q ldr \dst_q, [Htable, #96] .endm .macro load_h6 dst, dst_q ldr \dst_q, [Htable, #128] .endm .macro load_h7 dst, dst_q ldr \dst_q, [Htable, #144] .endm .macro load_h8 dst, dst_q ldr \dst_q, [Htable, #176] .endm .macro load_h12 dst, dst_q ldr \dst_q, [Htable, #16] .endm .macro load_h34 dst, dst_q ldr \dst_q, [Htable, #64] .endm .macro load_h56 dst, dst_q ldr \dst_q, [Htable, #112] .endm .macro load_h78 dst, dst_q ldr \dst_q, [Htable, #160] .endm .macro load_full_htable load_h1 Ht1, Ht1q load_h2 Ht2, Ht2q load_h3 Ht3, Ht3q load_h4 Ht4, Ht4q load_h5 Ht5, Ht5q load_h6 Ht6, Ht6q load_h12 Ht12, Ht12q load_h34 Ht34, Ht34q load_h56 Ht56, Ht56q .endm .macro load_htable_12 load_h1 Ht1, Ht1q load_h2 Ht2, Ht2q load_h12 Ht12, Ht12q .endm .macro load_htable_34 load_h3 Ht3, Ht3q load_h4 Ht4, Ht4q load_h34 Ht34, Ht34q .endm .macro load_htable_56 load_h5 Ht5, Ht5q load_h6 Ht6, Ht6q load_h56 Ht56, Ht56q .endm .macro load_htable_78 load_h7 Ht7, Ht7q load_h8 Ht8, Ht8q load_h78 Ht78, Ht78q .endm /********************************************************************/ /* Macros for GHASH udpate */ /********************************************************************/ .macro ghash_init_0 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_mid.1q, ghash_tmp.1d, \Hk_mid\().1d .endm .macro ghash_init_1 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_mid.1q, ghash_tmp.2d, \Hk_mid\().2d .endm .macro ghash_acc_0 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_tmp.1q, ghash_tmp.1d, \Hk_mid\().1d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_acc_1 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_tmp.1q, ghash_tmp.2d, \Hk_mid\().2d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_init_with_tag_0 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_mid.1q, ghash_tmp.1d, \Hk_mid\().1d .endm .macro ghash_init_with_tag_1 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_mid.1q, ghash_tmp.2d, \Hk_mid\().2d .endm .macro ghash_acc_with_tag_0 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_tmp.1q, ghash_tmp.1d, \Hk_mid\().1d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_acc_with_tag_1 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_tmp.1q, ghash_tmp.2d, \Hk_mid\().2d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_finalize tag eor modulo_tmp0.16b, ghash_lo.16b, ghash_hi.16b pmull modulo_tmp1.1q, ghash_hi.1d, ghash_mod.1d ext ghash_hi.16b, ghash_hi.16b, ghash_hi.16b, #8 eor ghash_mid.16b, ghash_mid.16b, modulo_tmp0.16b eor modulo_tmp1.16b, ghash_hi.16b, modulo_tmp1.16b eor ghash_mid.16b, ghash_mid.16b, modulo_tmp1.16b pmull ghash_hi.1q, ghash_mid.1d, ghash_mod.1d eor ghash_lo.16b, ghash_lo.16b, ghash_hi.16b ext ghash_mid.16b, ghash_mid.16b, ghash_mid.16b, #8 eor \tag\().16b, ghash_lo.16b, ghash_mid.16b ext \tag\().16b, \tag\().16b, \tag\().16b, #8 .endm .macro load_tag ldr tag_q, [tag_ptr] rev64 tag.16b, tag.16b .endm .macro prepare_ghash // Prepare constant for modular reduction movi ghash_mod.8b, #0xc2 shl ghash_mod_d, ghash_mod_d, #56 .endm /********************************************************************/ /* Core routine */ /********************************************************************/ .align 4 _aes_gcm_enc_kernel_slothy_base_128: aes_gcm_enc_kernel_slothy_base_128: #ifdef BORINGSSL_DISPATCH_TEST adrp x9,_BORINGSSL_function_hit@PAGE add x9, x9, _BORINGSSL_function_hit@PAGEOFF mov w10, #1 strb w10, [x9,#15] // kFlag_aes_gcm_slothy #endif AARCH64_VALID_CALL_TARGET sub sp, sp, #STACK_SIZE Lenc_preamble_start: save_gprs save_vregs lsr byte_len, len_bits, #3 load_round_keys load_tag load_iv prepare_loop_counts prepare_ghash load_htable_12 load_htable_34 Lenc_preamble_end: cbz count, Lloop_unrolled_end cmp count, #1 b.eq Lloop_unrolled_start_iter_1 // Instructions: 115 // Expected cycles: 63 // Expected IPC: 1.83 // // Cycle bound: 63.0 // IPC bound: 1.83 // // Wall time: 1.82s // User time: 1.82s // // ---------------------- cycle (expected) ----------------------> // 0 25 50 // |------------------------|------------------------|------------ ldr q1, [x0, #48] // *.............................................................. add w7, w13, #0 // *.............................................................. add w14, w13, #2 // *.............................................................. add w17, w13, #3 // *.............................................................. ldr q9, [x0], #(4*16) // .*............................................................. add w21, w13, #1 // .*............................................................. add w13, w13, #UNROLL // .*............................................................. rev w7, w7 // .*............................................................. ldr q4, [x0, #-48] // ..*............................................................ rev w14, w14 // ..*............................................................ rev w17, w17 // ..*............................................................ orr x7, x12, x7, lsl #32 // ..*............................................................ ldr q5, [x0, #-32] // ...*........................................................... orr x14, x12, x14, lsl #32 // ...*........................................................... orr x17, x12, x17, lsl #32 // ...*........................................................... stp x11, x7, [sp, #STACK_BASE_AES_ST] // ...*........................................................... // @slothy:writes=stack_0 eor v6.16B, v1.16B, v28.16B // ....*.......................................................... rev w7, w21 // ....*.......................................................... stp x11, x14, [sp, #STACK_BASE_AES_ST + 32] // ....*.......................................................... // @slothy:writes=stack_2 add w14, w13, #2 // ....*.......................................................... eor v3.16B, v9.16B, v28.16B // .....*......................................................... ldr q29, [sp, #STACK_BASE_AES_ST] // .....*......................................................... // @slothy:reads=stack_0 orr x7, x12, x7, lsl #32 // .....*......................................................... add w21, w13, #1 // .....*......................................................... ldr q31, [sp, #STACK_BASE_AES_ST + 32] // ......*........................................................ // @slothy:reads=stack_2 rev w14, w14 // ......*........................................................ add w27, w13, #0 // ......*........................................................ add w10, w13, #3 // ......*........................................................ eor v4.16B, v4.16B, v28.16B // .......*....................................................... orr x14, x12, x14, lsl #32 // .......*....................................................... rev w27, w27 // .......*....................................................... rev w10, w10 // .......*....................................................... eor v5.16B, v5.16B, v28.16B // ........*...................................................... rev w25, w21 // ........*...................................................... orr x21, x12, x27, lsl #32 // ........*...................................................... orr x27, x12, x10, lsl #32 // ........*...................................................... aesr v29.16b, v18.16b // .........*..................................................... stp x11, x17, [sp, #STACK_BASE_AES_ST + 48] // .........*..................................................... // @slothy:writes=stack_3 stp x11, x7, [sp, #STACK_BASE_AES_ST + 16] // .........*..................................................... // @slothy:writes=stack_1 aesr v31.16b, v18.16b // ..........*.................................................... aesr v29.16b, v19.16b // ...........*................................................... ldr q8, [sp, #STACK_BASE_AES_ST + 48] // ...........*................................................... // @slothy:reads=stack_3 stp x11, x21, [sp, #STACK_BASE_AES_ST] // ...........*................................................... // @slothy:writes=stack_0 aesr v31.16b, v19.16b // ............*.................................................. ldr q2, [sp, #STACK_BASE_AES_ST + 16] // ............*.................................................. // @slothy:reads=stack_1 aesr v29.16b, v20.16b // .............*................................................. ldr q9, [sp, #STACK_BASE_AES_ST] // .............*................................................. // @slothy:reads=stack_0 stp x11, x27, [sp, #STACK_BASE_AES_ST + 48] // .............*................................................. // @slothy:writes=stack_3 aesr v31.16b, v20.16b // ..............*................................................ aesr v29.16b, v21.16b // ...............*............................................... ldr q1, [sp, #STACK_BASE_AES_ST + 48] // ...............*............................................... // @slothy:reads=stack_3 stp x11, x14, [sp, #STACK_BASE_AES_ST + 32] // ...............*............................................... // @slothy:writes=stack_2 aesr v8.16b, v18.16b // ................*.............................................. aesr v29.16b, v22.16b // .................*............................................. aesr v8.16b, v19.16b // ..................*............................................ aesr v29.16b, v23.16b // ...................*........................................... aesr v8.16b, v20.16b // ....................*.......................................... aesr v29.16b, v24.16b // .....................*......................................... aesr v8.16b, v21.16b // ......................*........................................ aesr v29.16b, v25.16b // .......................*....................................... aesr v8.16b, v22.16b // ........................*...................................... aesr v29.16b, v26.16b // .........................*..................................... aesr v8.16b, v23.16b // ..........................*.................................... aese v29.16b, v27.16b // ...........................*................................... aesr v8.16b, v24.16b // ............................*.................................. eor v3.16B, v3.16B, v29.16B // .............................*................................. aesr v2.16b, v18.16b // .............................*................................. aesr v8.16b, v25.16b // ..............................*................................ aesr v2.16b, v19.16b // ...............................*............................... rev64 v29.16B, v3.16B // ...............................*............................... str q3, [x2], #(4*16) // ................................*.............................. aesr v8.16b, v26.16b // ................................*.............................. eor v11.16B, v29.16B, v30.16B // .................................*............................. aesr v2.16b, v20.16b // .................................*............................. aese v8.16b, v27.16b // ..................................*............................ aesr v2.16b, v21.16b // ...................................*........................... eor v6.16B, v6.16B, v8.16B // ....................................*.......................... aesr v31.16b, v21.16b // ....................................*.......................... aesr v2.16b, v22.16b // .....................................*......................... aesr v31.16b, v22.16b // ......................................*........................ rev64 v3.16B, v6.16B // ......................................*........................ str q6, [x2, #-16] // .......................................*....................... aesr v2.16b, v23.16b // .......................................*....................... aesr v31.16b, v23.16b // ........................................*...................... mov d6, v3.d[1] // ........................................*...................... aesr v2.16b, v24.16b // .........................................*..................... aesr v31.16b, v24.16b // ..........................................*.................... eor v0.8B, v6.8B, v3.8B // ..........................................*.................... aesr v2.16b, v25.16b // ...........................................*................... aesr v31.16b, v25.16b // ............................................*.................. aesr v2.16b, v26.16b // .............................................*................. aesr v31.16b, v26.16b // ..............................................*................ aese v2.16b, v27.16b // ...............................................*............... aese v31.16b, v27.16b // ................................................*.............. eor v29.16B, v4.16B, v2.16B // .................................................*............. pmull v4.1q, v3.1d, v12.1d // .................................................*............. eor v6.16B, v5.16B, v31.16B // ..................................................*............ aesr v9.16b, v18.16b // ..................................................*............ pmull2 v31.1q, v3.2d, v12.2d // ...................................................*........... rev64 v5.16B, v29.16B // ...................................................*........... str q29, [x2, #-48] // ....................................................*.......... aesr v9.16b, v19.16b // ....................................................*.......... aesr v1.16b, v18.16b // .....................................................*......... rev64 v29.16B, v6.16B // .....................................................*......... aesr v9.16b, v20.16b // ......................................................*........ ext v30.16B, v29.16B, v29.16B, #8 // .......................................................*....... pmull2 v3.1q, v5.2d, v15.2d // .......................................................*....... aesr v9.16b, v21.16b // ........................................................*...... aesr v1.16b, v19.16b // .........................................................*..... eor v2.16B, v30.16B, v29.16B // .........................................................*..... aesr v9.16b, v22.16b // ..........................................................*.... aesr v1.16b, v20.16b // ...........................................................*... aesr v9.16b, v23.16b // ............................................................*.. pmull2 v10.1q, v29.2d, v13.2d // .............................................................*. aesr v9.16b, v24.16b // ..............................................................* // ---------------------- cycle (expected) ----------------------> // 0 25 50 // |------------------------|------------------------|------------ // add w27, w13, #2 // *.............................................................. // add w23, w13, #0 // *.............................................................. // add w7, w13, #3 // *.............................................................. // add w8, w13, #1 // .*............................................................. // rev w21, w23 // .*............................................................. // orr x14, x12, x21, lsl #32 // ..*............................................................ // rev w25, w27 // ..*............................................................ // orr x22, x12, x25, lsl #32 // ...*........................................................... // stp x11, x14, [sp, #STACK_BASE_AES_ST] // ...*........................................................... // rev w21, w7 // ..*............................................................ // orr x14, x12, x21, lsl #32 // ...*........................................................... // stp x11, x22, [sp, #STACK_BASE_AES_ST + 32] // ....*.......................................................... // stp x11, x14, [sp, #STACK_BASE_AES_ST + 48] // .........*..................................................... // ldr q9, [sp, #STACK_BASE_AES_ST] // .....*......................................................... // ldr q1, [sp, #STACK_BASE_AES_ST + 48] // ...........*................................................... // aesr v9.16b, v18.16b // .........*..................................................... // aesr v1.16b, v18.16b // ................*.............................................. // aesr v9.16b, v19.16b // ...........*................................................... // aesr v9.16b, v20.16b // .............*................................................. // aesr v9.16b, v21.16b // ...............*............................................... // aesr v1.16b, v19.16b // ..................*............................................ // aesr v9.16b, v22.16b // .................*............................................. // aesr v1.16b, v20.16b // ....................*.......................................... // aesr v9.16b, v23.16b // ...................*........................................... // aesr v9.16b, v24.16b // .....................*......................................... // rev w25, w8 // ....*.......................................................... // aesr v1.16b, v21.16b // ......................*........................................ // add w13, w13, #UNROLL // .*............................................................. // orr x26, x12, x25, lsl #32 // .....*......................................................... // add w27, w13, #2 // ....*.......................................................... // stp x11, x26, [sp, #STACK_BASE_AES_ST + 16] // .........*..................................................... // aesr v1.16b, v22.16b // ........................*...................................... // add w23, w13, #0 // ......*........................................................ // add w7, w13, #3 // ......*........................................................ // ldr q8, [sp, #STACK_BASE_AES_ST + 16] // ............*.................................................. // aesr v1.16b, v23.16b // ..........................*.................................... // add w8, w13, #1 // .....*......................................................... // rev w21, w23 // .......*....................................................... // aesr v1.16b, v24.16b // ............................*.................................. // orr x14, x12, x21, lsl #32 // ........*...................................................... // ldr q31, [x0, #48] // *.............................................................. // rev w25, w27 // ......*........................................................ // aesr v8.16b, v18.16b // .............................*................................. // orr x22, x12, x25, lsl #32 // .......*....................................................... // aesr v1.16b, v25.16b // ..............................*................................ // ldr q10, [sp, #STACK_BASE_AES_ST + 32] // ......*........................................................ // aesr v8.16b, v19.16b // ...............................*............................... // stp x11, x14, [sp, #STACK_BASE_AES_ST] // ...........*................................................... // ldr q2, [x0], #(4*16) // .*............................................................. // aesr v1.16b, v26.16b // ................................*.............................. // aesr v8.16b, v20.16b // .................................*............................. // aese v1.16b, v27.16b // ..................................*............................ // eor v4.16B, v31.16B, v28.16B // ....*.......................................................... // rev w21, w7 // .......*....................................................... // aesr v8.16b, v21.16b // ...................................*........................... // ldr q29, [x0, #-48] // ..*............................................................ // orr x14, x12, x21, lsl #32 // ........*...................................................... // eor v5.16B, v4.16B, v1.16B // ....................................*.......................... // aesr v8.16b, v22.16b // .....................................*......................... // stp x11, x22, [sp, #STACK_BASE_AES_ST + 32] // ...............*............................................... // aesr v8.16b, v23.16b // .......................................*....................... // aesr v10.16b, v18.16b // ..........*.................................................... // rev64 v1.16B, v5.16B // ......................................*........................ // aesr v8.16b, v24.16b // .........................................*..................... // str q5, [x2, #48] // .......................................*....................... // aesr v8.16b, v25.16b // ...........................................*................... // eor v2.16B, v2.16B, v28.16B // .....*......................................................... // aesr v9.16b, v25.16b // .......................*....................................... // aesr v8.16b, v26.16b // .............................................*................. // eor v11.16B, v29.16B, v28.16B // .......*....................................................... // aesr v9.16b, v26.16b // .........................*..................................... // aese v8.16b, v27.16b // ...............................................*............... // mov d0, v1.d[1] // ........................................*...................... // eor v8.16B, v11.16B, v8.16B // .................................................*............. // aesr v10.16b, v19.16b // ............*.................................................. // aese v9.16b, v27.16b // ...........................*................................... // aesr v10.16b, v20.16b // ..............*................................................ // eor v0.8B, v0.8B, v1.8B // ..........................................*.................... // stp x11, x14, [sp, #STACK_BASE_AES_ST + 48] // .............*................................................. // eor v29.16B, v2.16B, v9.16B // .............................*................................. // ldr q9, [sp, #STACK_BASE_AES_ST] // .............*................................................. // aesr v10.16b, v21.16b // ....................................*.......................... // pmull2 v31.1q, v1.2d, v12.2d // ...................................................*........... // aesr v10.16b, v22.16b // ......................................*........................ // pmull v4.1q, v1.1d, v12.1d // .................................................*............. // rev64 v5.16B, v8.16B // ...................................................*........... // aesr v10.16b, v23.16b // ........................................*...................... // ldr q1, [sp, #STACK_BASE_AES_ST + 48] // ...............*............................................... // aesr v9.16b, v18.16b // ..................................................*............ // aesr v10.16b, v24.16b // ..........................................*.................... // ldr q2, [x0, #-32] // ...*........................................................... // str q8, [x2, #16] // ....................................................*.......... // aesr v10.16b, v25.16b // ............................................*.................. // aesr v1.16b, v18.16b // .....................................................*......... // aesr v10.16b, v26.16b // ..............................................*................ // rev64 v8.16B, v29.16B // ...............................*............................... // eor v6.16B, v2.16B, v28.16B // ........*...................................................... // aesr v9.16b, v19.16b // ....................................................*.......... // aese v10.16b, v27.16b // ................................................*.............. // aesr v9.16b, v20.16b // ......................................................*........ // str q29, [x2], #(4*16) // ................................*.............................. // pmull2 v3.1q, v5.2d, v15.2d // .......................................................*....... // eor v6.16B, v6.16B, v10.16B // ..................................................*............ // aesr v9.16b, v21.16b // ........................................................*...... // aesr v1.16b, v19.16b // .........................................................*..... // rev64 v29.16B, v6.16B // .....................................................*......... // aesr v9.16b, v22.16b // ..........................................................*.... // aesr v1.16b, v20.16b // ...........................................................*... // ext v11.16B, v29.16B, v29.16B, #8 // .......................................................*....... // aesr v9.16b, v23.16b // ............................................................*.. // eor v2.16B, v11.16B, v29.16B // .........................................................*..... // pmull2 v10.1q, v29.2d, v13.2d // .............................................................*. // eor v11.16B, v8.16B, v30.16B // .................................*............................. // aesr v9.16b, v24.16b // ..............................................................* // rev w25, w8 // ........*...................................................... sub count, count, #2 cbz count, Lloop_unrolled_start_postamble Lloop_unrolled_start: // Instructions: 122 // Expected cycles: 54 // Expected IPC: 2.26 // // Cycle bound: 54.0 // IPC bound: 2.26 // // Wall time: 1546.50s // User time: 1546.50s // // ----------------- cycle (expected) ------------------> // 0 25 50 // |------------------------|------------------------|--- str q6, [x2, #-32] // l..................................................... aesr v1.16b, v21.16b // *..................................................... add w13, w13, #UNROLL // *..................................................... orr x26, x12, x25, lsl #32 // *..................................................... mov d6, v5.d[1] // .l.................................................... pmull v0.1q, v0.1d, v14.1d // .l.................................................... add w27, w13, #2 // .e.................................................... stp x11, x26, [sp, #STACK_BASE_AES_ST + 16] // .*.................................................... // @slothy:writes=stack_1 aesr v1.16b, v22.16b // ..*................................................... eor v30.16B, v31.16B, v10.16B // ..l................................................... add w23, w13, #0 // ..e................................................... add w7, w13, #3 // ..e................................................... pmull2 v10.1q, v2.2d, v14.2d // ...l.................................................. ldr q8, [sp, #STACK_BASE_AES_ST + 16] // ...*.................................................. // @slothy:reads=stack_1 eor v6.8B, v6.8B, v5.8B // ....l................................................. aesr v1.16b, v23.16b // ....*................................................. add w8, w13, #1 // ....e................................................. eor v10.16B, v0.16B, v10.16B // .....l................................................ pmull v0.1q, v29.1d, v13.1d // .....l................................................ rev w21, w23 // .....e................................................ aesr v1.16b, v24.16b // ......*............................................... orr x14, x12, x21, lsl #32 // ......e............................................... ldr q31, [x0, #48] // .......*.............................................. pmull v2.1q, v6.1d, v17.1d // .......l.............................................. rev w25, w27 // .......e.............................................. eor v6.16B, v30.16B, v3.16B // ........l............................................. aesr v8.16b, v18.16b // ........*............................................. orr x22, x12, x25, lsl #32 // ........e............................................. eor v3.16B, v10.16B, v2.16B // .........l............................................ aesr v1.16b, v25.16b // .........*............................................ ldr q10, [sp, #STACK_BASE_AES_ST + 32] // ..........*........................................... // @slothy:reads=stack_2 aesr v8.16b, v19.16b // ..........*........................................... stp x11, x14, [sp, #STACK_BASE_AES_ST] // ..........e........................................... // @slothy:writes=stack_0 ldr q2, [x0], #(4*16) // ...........*.......................................... aesr v1.16b, v26.16b // ...........*.......................................... eor v0.16B, v4.16B, v0.16B // ............l......................................... aesr v8.16b, v20.16b // ............*......................................... aese v1.16b, v27.16b // .............*........................................ eor v4.16B, v31.16B, v28.16B // .............*........................................ rev w21, w7 // .............e........................................ aesr v8.16b, v21.16b // ..............*....................................... ldr q29, [x0, #-48] // ..............*....................................... orr x14, x12, x21, lsl #32 // ..............e....................................... pmull v31.1q, v5.1d, v15.1d // ...............l...................................... eor v5.16B, v4.16B, v1.16B // ...............*...................................... aesr v8.16b, v22.16b // ................*..................................... ext v1.16B, v11.16B, v11.16B, #8 // ................l..................................... stp x11, x22, [sp, #STACK_BASE_AES_ST + 32] // ................e..................................... // @slothy:writes=stack_2 eor v30.16B, v0.16B, v31.16B // .................l.................................... pmull v31.1q, v11.1d, v16.1d // .................l.................................... aesr v8.16b, v23.16b // ..................*................................... eor v4.16B, v1.16B, v11.16B // ..................l................................... aesr v10.16b, v18.16b // ...................*.................................. rev64 v1.16B, v5.16B // ...................*.................................. eor v30.16B, v30.16B, v31.16B // ....................l................................. aesr v8.16b, v24.16b // ....................*................................. pmull2 v0.1q, v11.2d, v16.2d // .....................l................................ str q5, [x2, #48] // .....................*................................ aesr v8.16b, v25.16b // ......................*............................... eor v2.16B, v2.16B, v28.16B // ......................*............................... eor v5.16B, v6.16B, v0.16B // .......................l.............................. aesr v9.16b, v25.16b // .......................*.............................. aesr v8.16b, v26.16b // ........................*............................. eor v11.16B, v29.16B, v28.16B // ........................*............................. ext v31.16B, v5.16B, v5.16B, #8 // .........................l............................ aesr v9.16b, v26.16b // .........................*............................ eor v6.16B, v30.16B, v5.16B // ..........................l........................... aese v8.16b, v27.16b // ..........................*........................... pmull2 v29.1q, v4.2d, v17.2d // ...........................l.......................... mov d0, v1.d[1] // ...........................*.......................... eor v8.16B, v11.16B, v8.16B // ............................*......................... aesr v10.16b, v19.16b // ............................*......................... aese v9.16b, v27.16b // .............................*........................ eor v4.16B, v3.16B, v29.16B // .............................l........................ aesr v10.16b, v20.16b // ..............................*....................... eor v0.8B, v0.8B, v1.8B // ..............................*....................... stp x11, x14, [sp, #STACK_BASE_AES_ST + 48] // ..............................e....................... // @slothy:writes=stack_3 eor v29.16B, v2.16B, v9.16B // ...............................*...................... pmull v11.1q, v5.1d, v7.1d // ...............................l...................... ldr q9, [sp, #STACK_BASE_AES_ST] // ................................e..................... // @slothy:reads=stack_0 aesr v10.16b, v21.16b // ................................*..................... eor v11.16B, v31.16B, v11.16B // .................................l.................... pmull2 v31.1q, v1.2d, v12.2d // .................................*.................... aesr v10.16b, v22.16b // ..................................*................... eor v6.16B, v4.16B, v6.16B // ..................................l................... pmull v4.1q, v1.1d, v12.1d // ...................................*.................. rev64 v5.16B, v8.16B // ...................................*.................. aesr v10.16b, v23.16b // ....................................*................. ldr q1, [sp, #STACK_BASE_AES_ST + 48] // ....................................e................. // @slothy:reads=stack_3 eor v11.16B, v6.16B, v11.16B // .....................................l................ aesr v9.16b, v18.16b // .....................................e................ aesr v10.16b, v24.16b // ......................................*............... pmull v6.1q, v11.1d, v7.1d // .......................................l.............. ldr q2, [x0, #-32] // .......................................*.............. str q8, [x2, #16] // ........................................*............. aesr v10.16b, v25.16b // ........................................*............. eor v30.16B, v30.16B, v6.16B // .........................................l............ aesr v1.16b, v18.16b // .........................................e............ aesr v10.16b, v26.16b // ..........................................*........... rev64 v8.16B, v29.16B // ..........................................*........... eor v6.16B, v2.16B, v28.16B // ...........................................*.......... aesr v9.16b, v19.16b // ...........................................e.......... aese v10.16b, v27.16b // ............................................*......... ext v11.16B, v11.16B, v11.16B, #8 // ............................................l......... aesr v9.16b, v20.16b // .............................................e........ str q29, [x2], #(4*16) // .............................................*........ pmull2 v3.1q, v5.2d, v15.2d // ..............................................*....... eor v6.16B, v6.16B, v10.16B // ..............................................*....... aesr v9.16b, v21.16b // ...............................................e...... aesr v1.16b, v19.16b // ................................................e..... rev64 v29.16B, v6.16B // ................................................*..... eor v30.16B, v30.16B, v11.16B // .................................................l.... aesr v9.16b, v22.16b // .................................................e.... aesr v1.16b, v20.16b // ..................................................e... ext v11.16B, v29.16B, v29.16B, #8 // ..................................................*... ext v30.16B, v30.16B, v30.16B, #8 // ...................................................l.. aesr v9.16b, v23.16b // ...................................................e.. eor v2.16B, v11.16B, v29.16B // ....................................................*. pmull2 v10.1q, v29.2d, v13.2d // ....................................................*. eor v11.16B, v8.16B, v30.16B // .....................................................* aesr v9.16b, v24.16b // .....................................................e rev w25, w8 // .....................................................e // ---------------------------------------------------------------------- cycle (expected) ----------------------------------------------------------------------> // 0 25 50 75 100 125 150 // |------------------------|------------------------|------------------------|------------------------|------------------------|------------------------|-------- // ldr q29, [x0, #(3*16)] // ......~..............................................'......*..............................................'......~............................................ // add w14, w13, #3 // .e...................................................'.~...................................................'.~................................................. // rev w14, w14 // ............e........................................'............~........................................'............~...................................... // orr x14, x12, x14, lsl #32 // .............e.......................................'.............~.......................................'.............~..................................... // stp x11, x14, [sp, #(STACK_BASE_AES_ST + 3*16)] // .............................e.......................'.............................~.......................'.............................~..................... // ldr q0, [sp, #(STACK_BASE_AES_ST + 3*16)] // ...................................e.................'...................................~.................'...................................~............... // aesr v0.16b, v18.16b // ........................................e............'........................................~............'........................................~.......... // aesr v0.16b, v19.16b // ...............................................e.....'...............................................~.....'...............................................~... // aesr v0.16b, v20.16b // .................................................e...'.................................................~...'.................................................~. // aesr v0.16b, v21.16b // .....................................................*.....................................................~................................................... // aesr v0.16b, v22.16b // .~...................................................'.*...................................................'.~................................................. // aesr v0.16b, v23.16b // ...~.................................................'...*.................................................'...~............................................... // aesr v0.16b, v24.16b // .....~...............................................'.....*...............................................'.....~............................................. // aesr v0.16b, v25.16b // ........~............................................'........*............................................'........~.......................................... // aesr v0.16b, v26.16b // ..........~..........................................'..........*..........................................'..........~........................................ // aese v0.16b, v27.16b // ............~........................................'............*........................................'............~...................................... // eor v29.16b, v29.16b, v28.16b // ............~........................................'............*........................................'............~...................................... // eor v0.16b, v29.16b, v0.16b // ..............~......................................'..............*......................................'..............~.................................... // str q0, [x2, #(3*16)] // ....................~................................'....................*................................'....................~.............................. // rev64 v0.16b, v0.16b // ..................~..................................'..................*..................................'..................~................................ // pmull v8.1q, v0.1d, v12.1d // ..................................~..................'..................................*..................'..................................~................ // pmull2 v9.1q, v0.2d, v12.2d // ................................~....................'................................*....................'................................~.................. // mov d11, v0.d[1] // ..........................~..........................'..........................*..........................'..........................~........................ // eor v11.8b, v11.8b, v0.8b // .............................~.......................'.............................*.......................'.............................~..................... // pmull v10.1q, v11.1d, v14.1d // ~....................................................'~....................................................'l.................................................. // ldr q29, [x0, #(2*16)] // ......................................~..............'......................................*..............'......................................~............ // add w14, w13, #2 // e....................................................'~....................................................'~.................................................. // rev w14, w14 // ......e..............................................'......~..............................................'......~............................................ // orr x14, x12, x14, lsl #32 // .......e.............................................'.......~.............................................'.......~........................................... // stp x11, x14, [sp, #(STACK_BASE_AES_ST + 2*16)] // ...............e.....................................'...............~.....................................'...............~................................... // ldr q0, [sp, #(STACK_BASE_AES_ST + 2*16)] // .........~...........................................'.........*...........................................'.........~......................................... // aesr v0.16b, v18.16b // ..................~..................................'..................*..................................'..................~................................ // aesr v0.16b, v19.16b // ...........................~.........................'...........................*.........................'...........................~....................... // aesr v0.16b, v20.16b // .............................~.......................'.............................*.......................'.............................~..................... // aesr v0.16b, v21.16b // ...............................~.....................'...............................*.....................'...............................~................... // aesr v0.16b, v22.16b // .................................~...................'.................................*...................'.................................~................. // aesr v0.16b, v23.16b // ...................................~.................'...................................*.................'...................................~............... // aesr v0.16b, v24.16b // .....................................~...............'.....................................*...............'.....................................~............. // aesr v0.16b, v25.16b // .......................................~.............'.......................................*.............'.......................................~........... // aesr v0.16b, v26.16b // .........................................~...........'.........................................*...........'.........................................~......... // aese v0.16b, v27.16b // ...........................................~.........'...........................................*.........'...........................................~....... // eor v29.16b, v29.16b, v28.16b // ..........................................~..........'..........................................*..........'..........................................~........ // eor v0.16b, v29.16b, v0.16b // .............................................~.......'.............................................*.......'.............................................~..... // str q0, [x2, #(2*16)] // .....................................................~.....................................................l................................................... // rev64 v0.16b, v0.16b // ...............................................~.....'...............................................*.....'...............................................~... // pmull v11.1q, v0.1d, v13.1d // ....~................................................'....~................................................'....l.............................................. // eor v8.16b, v8.16b, v11.16b // ...........~.........................................'...........~.........................................'...........l....................................... // pmull2 v11.1q, v0.2d, v13.2d // ...................................................~.'...................................................*.'................................................... // eor v9.16b, v9.16b, v11.16b // .~...................................................'.~...................................................'.l................................................. // ext v11.16b, v0.16b, v0.16b, #8 // .................................................~...'.................................................*...'.................................................~. // eor v11.16b, v11.16b, v0.16b // ...................................................~.'...................................................*.'................................................... // pmull2 v11.1q, v11.2d, v14.2d // ..~..................................................'..~..................................................'..l................................................ // eor v10.16b, v10.16b, v11.16b // ....~................................................'....~................................................'....l.............................................. // ldr q29, [x0, #(1*16)] // .............~.......................................'.............*.......................................'.............~..................................... // add w14, w13, #1 // ...e.................................................'...~.................................................'...~............................................... // rev w14, w14 // ....................................................e'....................................................~'................................................... // orr x14, x12, x14, lsl #32 // .....................................................*.....................................................~................................................... // stp x11, x14, [sp, #(STACK_BASE_AES_ST + 1*16)] // ~....................................................'*....................................................'~.................................................. // ldr q0, [sp, #(STACK_BASE_AES_ST + 1*16)] // ..~..................................................'..*..................................................'..~................................................ // aesr v0.16b, v18.16b // .......~.............................................'.......*.............................................'.......~........................................... // aesr v0.16b, v19.16b // .........~...........................................'.........*...........................................'.........~......................................... // aesr v0.16b, v20.16b // ...........~.........................................'...........*.........................................'...........~....................................... // aesr v0.16b, v21.16b // .............~.......................................'.............*.......................................'.............~..................................... // aesr v0.16b, v22.16b // ...............~.....................................'...............*.....................................'...............~................................... // aesr v0.16b, v23.16b // .................~...................................'.................*...................................'.................~................................. // aesr v0.16b, v24.16b // ...................~.................................'...................*.................................'...................~............................... // aesr v0.16b, v25.16b // .....................~...............................'.....................*...............................'.....................~............................. // aesr v0.16b, v26.16b // .......................~.............................'.......................*.............................'.......................~........................... // aese v0.16b, v27.16b // .........................~...........................'.........................*...........................'.........................~......................... // eor v29.16b, v29.16b, v28.16b // .......................~.............................'.......................*.............................'.......................~........................... // eor v0.16b, v29.16b, v0.16b // ...........................~.........................'...........................*.........................'...........................~....................... // str q0, [x2, #(1*16)] // .......................................~.............'.......................................*.............'.......................................~........... // rev64 v0.16b, v0.16b // ..................................~..................'..................................*..................'..................................~................ // pmull v11.1q, v0.1d, v15.1d // ..............~......................................'..............~......................................'..............l.................................... // eor v8.16b, v8.16b, v11.16b // ................~....................................'................~....................................'................l.................................. // pmull2 v11.1q, v0.2d, v15.2d // .............................................~.......'.............................................*.......'.............................................~..... // eor v9.16b, v9.16b, v11.16b // .......~.............................................'.......~.............................................'.......l........................................... // mov d11, v0.d[1] // ~....................................................'~....................................................'l.................................................. // eor v11.8b, v11.8b, v0.8b // ...~.................................................'...~.................................................'...l............................................... // pmull v11.1q, v11.1d, v17.1d // ......~..............................................'......~..............................................'......l............................................ // eor v10.16b, v10.16b, v11.16b // ........~............................................'........~............................................'........l.......................................... // ldr q29, [x0], #(4*16) // ..........~..........................................'..........*..........................................'..........~........................................ // add w14, w13, #0 // .e...................................................'.~...................................................'.~................................................. // rev w14, w14 // ....e................................................'....~................................................'....~.............................................. // orr x14, x12, x14, lsl #32 // .....e...............................................'.....~...............................................'.....~............................................. // stp x11, x14, [sp, #(STACK_BASE_AES_ST + 0*16)] // .........e...........................................'.........~...........................................'.........~......................................... // ldr q0, [sp, #(STACK_BASE_AES_ST + 0*16)] // ...............................e.....................'...............................~.....................'...............................~................... // aesr v0.16b, v18.16b // ....................................e................'....................................~................'....................................~.............. // aesr v0.16b, v19.16b // ..........................................e..........'..........................................~..........'..........................................~........ // aesr v0.16b, v20.16b // ............................................e........'............................................~........'............................................~...... // aesr v0.16b, v21.16b // ..............................................e......'..............................................~......'..............................................~.... // aesr v0.16b, v22.16b // ................................................e....'................................................~....'................................................~.. // aesr v0.16b, v23.16b // ..................................................e..'..................................................~..'................................................... // aesr v0.16b, v24.16b // ....................................................e'....................................................~'................................................... // aesr v0.16b, v25.16b // ......................~..............................'......................*..............................'......................~............................ // aesr v0.16b, v26.16b // ........................~............................'........................*............................'........................~.......................... // aese v0.16b, v27.16b // ............................~........................'............................*........................'............................~...................... // eor v29.16b, v29.16b, v28.16b // .....................~...............................'.....................*...............................'.....................~............................. // eor v0.16b, v29.16b, v0.16b // ..............................~......................'..............................*......................'..............................~.................... // str q0, [x2], #(4*16) // ............................................~........'............................................*........'............................................~...... // rev64 v0.16b, v0.16b // .........................................~...........'.........................................*...........'.........................................~......... // eor v0.16b, v0.16b, v30.16b // ....................................................~'....................................................*'................................................... // pmull v11.1q, v0.1d, v16.1d // ................~....................................'................~....................................'................l.................................. // eor v8.16b, v8.16b, v11.16b // ...................~.................................'...................~.................................'...................l............................... // pmull2 v11.1q, v0.2d, v16.2d // ....................~................................'....................~................................'....................l.............................. // eor v9.16b, v9.16b, v11.16b // ......................~..............................'......................~..............................'......................l............................ // ext v11.16b, v0.16b, v0.16b, #8 // ...............~.....................................'...............~.....................................'...............l................................... // eor v11.16b, v11.16b, v0.16b // .................~...................................'.................~...................................'.................l................................. // pmull2 v11.1q, v11.2d, v17.2d // ..........................~..........................'..........................~..........................'..........................l........................ // eor v10.16b, v10.16b, v11.16b // ............................~........................'............................~........................'............................l...................... // eor v0.16b, v8.16b, v9.16b // .........................~...........................'.........................~...........................'.........................l......................... // pmull v1.1q, v9.1d, v7.1d // ..............................~......................'..............................~......................'..............................l.................... // ext v9.16b, v9.16b, v9.16b, #8 // ........................~............................'........................~............................'........................l.......................... // eor v10.16b, v10.16b, v0.16b // .................................~...................'.................................~...................'.................................l................. // eor v1.16b, v9.16b, v1.16b // ................................~....................'................................~....................'................................l.................. // eor v10.16b, v10.16b, v1.16b // ....................................~................'....................................~................'....................................l.............. // pmull v9.1q, v10.1d, v7.1d // ......................................~..............'......................................~..............'......................................l............ // eor v8.16b, v8.16b, v9.16b // ........................................~............'........................................~............'........................................l.......... // ext v10.16b, v10.16b, v10.16b, #8 // ...........................................~.........'...........................................~.........'...........................................l....... // eor v30.16b, v8.16b, v10.16b // ................................................~....'................................................~....'................................................l.. // ext v30.16b, v30.16b, v30.16b, #8 // ..................................................~..'..................................................~..'..................................................l // add w13, w13, #UNROLL // .....................................................*.....................................................~................................................... sub count, count, #1 cbnz count, Lloop_unrolled_start Lloop_unrolled_start_postamble:// end of loop kernel // Instructions: 129 // Expected cycles: 69 // Expected IPC: 1.87 // // Cycle bound: 68.0 // IPC bound: 1.90 // // Wall time: 1800.09s // User time: 1800.09s // // ------------------------- cycle (expected) -------------------------> // 0 25 50 // |------------------------|------------------------|------------------ eor v31.16B, v31.16B, v10.16B // *.................................................................... aesr v1.16b, v21.16b // *.................................................................... add w13, w13, #UNROLL // *.................................................................... orr x7, x12, x25, lsl #32 // *.................................................................... pmull2 v30.1q, v11.2d, v16.2d // .*................................................................... mov d8, v5.d[1] // .*................................................................... stp x11, x7, [sp, #STACK_BASE_AES_ST + 16] // .*................................................................... // @slothy:writes=stack_1 eor v10.16B, v31.16B, v3.16B // ..*.................................................................. aesr v1.16b, v22.16b // ..*.................................................................. eor v3.8B, v8.8B, v5.8B // ...*................................................................. pmull v0.1q, v0.1d, v14.1d // ...*................................................................. eor v8.16B, v10.16B, v30.16B // ....*................................................................ aesr v1.16b, v23.16b // ....*................................................................ ldr q30, [x0, #16] // .....*............................................................... pmull v10.1q, v3.1d, v17.1d // .....*............................................................... pmull v31.1q, v8.1d, v7.1d // ......*.............................................................. str q6, [x2, #-32] // ......*.............................................................. pmull2 v2.1q, v2.2d, v14.2d // .......*............................................................. ext v6.16B, v11.16B, v11.16B, #8 // .......*............................................................. pmull v3.1q, v29.1d, v13.1d // ........*............................................................ ldr q29, [sp, #STACK_BASE_AES_ST + 32] // ........*............................................................ // @slothy:reads=stack_2 eor v2.16B, v0.16B, v2.16B // .........*........................................................... eor v0.16B, v6.16B, v11.16B // .........*........................................................... pmull v5.1q, v5.1d, v15.1d // ..........*.......................................................... eor v4.16B, v4.16B, v3.16B // ..........*.......................................................... eor v3.16B, v2.16B, v10.16B // ...........*......................................................... aesr v1.16b, v24.16b // ...........*......................................................... eor v4.16B, v4.16B, v5.16B // ............*........................................................ pmull v10.1q, v11.1d, v16.1d // ............*........................................................ ldr q5, [x0, #48] // .............*....................................................... aesr v1.16b, v25.16b // .............*....................................................... aesr v29.16b, v18.16b // ..............*...................................................... eor v11.16B, v4.16B, v10.16B // ..............*...................................................... aesr v1.16b, v26.16b // ...............*..................................................... ext v6.16B, v8.16B, v8.16B, #8 // ...............*..................................................... eor v10.16B, v11.16B, v8.16B // ................*.................................................... aesr v29.16b, v19.16b // ................*.................................................... aese v1.16b, v27.16b // .................*................................................... eor v31.16B, v6.16B, v31.16B // .................*................................................... ldr q2, [x0], #(4*16) // ..................*.................................................. pmull2 v6.1q, v0.2d, v17.2d // ..................*.................................................. eor v5.16B, v5.16B, v28.16B // ...................*................................................. aesr v29.16b, v20.16b // ...................*................................................. ldr q4, [sp, #STACK_BASE_AES_ST + 16] // ....................*................................................ // @slothy:reads=stack_1 aesr v9.16b, v25.16b // ....................*................................................ eor v0.16B, v5.16B, v1.16B // .....................*............................................... aesr v29.16b, v21.16b // .....................*............................................... eor v8.16B, v3.16B, v6.16B // ......................*.............................................. aesr v9.16b, v26.16b // ......................*.............................................. aesr v29.16b, v22.16b // .......................*............................................. rev64 v5.16B, v0.16B // .......................*............................................. eor v3.16B, v8.16B, v10.16B // ........................*............................................ aesr v4.16b, v18.16b // ........................*............................................ aesr v29.16b, v23.16b // .........................*........................................... mov d1, v5.d[1] // .........................*........................................... eor v3.16B, v3.16B, v31.16B // ..........................*.......................................... aesr v4.16b, v19.16b // ..........................*.......................................... aesr v29.16b, v24.16b // ...........................*......................................... ldr q6, [x0, #-32] // ...........................*......................................... aesr v4.16b, v20.16b // ............................*........................................ ext v10.16B, v3.16B, v3.16B, #8 // ............................*........................................ eor v31.8B, v1.8B, v5.8B // .............................*....................................... pmull v8.1q, v3.1d, v7.1d // .............................*....................................... eor v30.16B, v30.16B, v28.16B // ..............................*...................................... aesr v4.16b, v21.16b // ..............................*...................................... aese v9.16b, v27.16b // ...............................*..................................... eor v1.16B, v2.16B, v28.16B // ...............................*..................................... eor v8.16B, v11.16B, v8.16B // ................................*.................................... aesr v4.16b, v22.16b // ................................*.................................... aesr v29.16b, v25.16b // .................................*................................... eor v2.16B, v1.16B, v9.16B // .................................*................................... eor v8.16B, v8.16B, v10.16B // ..................................*.................................. aesr v4.16b, v23.16b // ..................................*.................................. rev64 v11.16B, v2.16B // ...................................*................................. aesr v29.16b, v26.16b // ...................................*................................. ext v3.16B, v8.16B, v8.16B, #8 // ....................................*................................ aesr v4.16b, v24.16b // ....................................*................................ aese v29.16b, v27.16b // .....................................*............................... eor v10.16B, v6.16B, v28.16B // .....................................*............................... aesr v4.16b, v25.16b // ......................................*.............................. eor v3.16B, v11.16B, v3.16B // ......................................*.............................. pmull v8.1q, v31.1d, v14.1d // .......................................*............................. eor v10.16B, v10.16B, v29.16B // .......................................*............................. ext v29.16B, v3.16B, v3.16B, #8 // ........................................*............................ aesr v4.16b, v26.16b // ........................................*............................ pmull2 v1.1q, v5.2d, v12.2d // .........................................*........................... rev64 v9.16B, v10.16B // .........................................*........................... eor v29.16B, v29.16B, v3.16B // ..........................................*.......................... aese v4.16b, v27.16b // ..........................................*.......................... pmull v11.1q, v9.1d, v13.1d // ...........................................*......................... ext v6.16B, v9.16B, v9.16B, #8 // ...........................................*......................... eor v31.16B, v30.16B, v4.16B // ............................................*........................ pmull v5.1q, v5.1d, v12.1d // ............................................*........................ eor v6.16B, v6.16B, v9.16B // .............................................*....................... pmull2 v30.1q, v9.2d, v13.2d // .............................................*....................... eor v11.16B, v5.16B, v11.16B // ..............................................*...................... rev64 v5.16B, v31.16B // ..............................................*...................... pmull2 v9.1q, v6.2d, v14.2d // ...............................................*..................... eor v4.16B, v1.16B, v30.16B // ...............................................*..................... pmull v6.1q, v3.1d, v16.1d // ................................................*.................... mov d30, v5.d[1] // ................................................*.................... eor v9.16B, v8.16B, v9.16B // .................................................*................... pmull2 v1.1q, v5.2d, v15.2d // .................................................*................... eor v8.8B, v30.8B, v5.8B // ..................................................*.................. pmull v5.1q, v5.1d, v15.1d // ..................................................*.................. eor v1.16B, v4.16B, v1.16B // ...................................................*................. pmull2 v4.1q, v3.2d, v16.2d // ...................................................*................. eor v30.16B, v11.16B, v5.16B // ....................................................*................ pmull v5.1q, v8.1d, v17.1d // ....................................................*................ eor v4.16B, v1.16B, v4.16B // .....................................................*............... pmull2 v8.1q, v29.2d, v17.2d // .....................................................*............... eor v29.16B, v9.16B, v5.16B // ......................................................*.............. eor v5.16B, v30.16B, v6.16B // ......................................................*.............. pmull v6.1q, v4.1d, v7.1d // .......................................................*............. ext v11.16B, v4.16B, v4.16B, #8 // .......................................................*............. eor v30.16B, v29.16B, v8.16B // ........................................................*............ eor v29.16B, v5.16B, v4.16B // ........................................................*............ str q0, [x2, #48] // .........................................................*........... str q10, [x2, #32] // .........................................................*........... eor v4.16B, v11.16B, v6.16B // ..........................................................*.......... eor v1.16B, v30.16B, v29.16B // ..........................................................*.......... str q2, [x2], #(4*16) // ...........................................................*......... str q31, [x2, #-48] // ...........................................................*......... eor v0.16B, v1.16B, v4.16B // ............................................................*........ ext v3.16B, v0.16B, v0.16B, #8 // ..............................................................*...... pmull v31.1q, v0.1d, v7.1d // ..............................................................*...... eor v10.16B, v5.16B, v31.16B // ................................................................*.... eor v31.16B, v10.16B, v3.16B // ..................................................................*.. ext v30.16B, v31.16B, v31.16B, #8 // ....................................................................* // ------------------------- cycle (expected) -------------------------> // 0 25 50 // |------------------------|------------------------|------------------ // str q6, [x2, #-32] // ......*.............................................................. // aesr v1.16b, v21.16b // *.................................................................... // add w13, w13, #UNROLL // *.................................................................... // orr x26, x12, x25, lsl #32 // *.................................................................... // mov d6, v5.d[1] // .*................................................................... // pmull v0.1q, v0.1d, v14.1d // ...*................................................................. // stp x11, x26, [sp, #STACK_BASE_AES_ST + 16] // .*................................................................... // aesr v1.16b, v22.16b // ..*.................................................................. // eor v30.16B, v31.16B, v10.16B // *.................................................................... // pmull2 v10.1q, v2.2d, v14.2d // .......*............................................................. // ldr q8, [sp, #STACK_BASE_AES_ST + 16] // ....................*................................................ // eor v6.8B, v6.8B, v5.8B // ...*................................................................. // aesr v1.16b, v23.16b // ....*................................................................ // eor v10.16B, v0.16B, v10.16B // .........*........................................................... // pmull v0.1q, v29.1d, v13.1d // ........*............................................................ // aesr v1.16b, v24.16b // ...........*......................................................... // ldr q31, [x0, #48] // .............*....................................................... // pmull v2.1q, v6.1d, v17.1d // .....*............................................................... // eor v6.16B, v30.16B, v3.16B // ..*.................................................................. // aesr v8.16b, v18.16b // ........................*............................................ // eor v3.16B, v10.16B, v2.16B // ...........*......................................................... // aesr v1.16b, v25.16b // .............*....................................................... // ldr q10, [sp, #STACK_BASE_AES_ST + 32] // ........*............................................................ // aesr v8.16b, v19.16b // ..........................*.......................................... // ldr q2, [x0], #(4*16) // ..................*.................................................. // aesr v1.16b, v26.16b // ...............*..................................................... // eor v0.16B, v4.16B, v0.16B // ..........*.......................................................... // aesr v8.16b, v20.16b // ............................*........................................ // aese v1.16b, v27.16b // .................*................................................... // eor v4.16B, v31.16B, v28.16B // ...................*................................................. // aesr v8.16b, v21.16b // ..............................*...................................... // ldr q29, [x0, #-48] // .....*............................................................... // pmull v31.1q, v5.1d, v15.1d // ..........*.......................................................... // eor v5.16B, v4.16B, v1.16B // .....................*............................................... // aesr v8.16b, v22.16b // ................................*.................................... // ext v1.16B, v11.16B, v11.16B, #8 // .......*............................................................. // eor v30.16B, v0.16B, v31.16B // ............*........................................................ // pmull v31.1q, v11.1d, v16.1d // ............*........................................................ // aesr v8.16b, v23.16b // ..................................*.................................. // eor v4.16B, v1.16B, v11.16B // .........*........................................................... // aesr v10.16b, v18.16b // ..............*...................................................... // rev64 v1.16B, v5.16B // .......................*............................................. // eor v30.16B, v30.16B, v31.16B // ..............*...................................................... // aesr v8.16b, v24.16b // ....................................*................................ // pmull2 v0.1q, v11.2d, v16.2d // .*................................................................... // str q5, [x2, #48] // .........................................................*........... // aesr v8.16b, v25.16b // ......................................*.............................. // eor v2.16B, v2.16B, v28.16B // ...............................*..................................... // eor v5.16B, v6.16B, v0.16B // ....*................................................................ // aesr v9.16b, v25.16b // ....................*................................................ // aesr v8.16b, v26.16b // ........................................*............................ // eor v11.16B, v29.16B, v28.16B // ..............................*...................................... // ext v31.16B, v5.16B, v5.16B, #8 // ...............*..................................................... // aesr v9.16b, v26.16b // ......................*.............................................. // eor v6.16B, v30.16B, v5.16B // ................*.................................................... // aese v8.16b, v27.16b // ..........................................*.......................... // pmull2 v29.1q, v4.2d, v17.2d // ..................*.................................................. // mov d0, v1.d[1] // .........................*........................................... // eor v8.16B, v11.16B, v8.16B // ............................................*........................ // aesr v10.16b, v19.16b // ................*.................................................... // aese v9.16b, v27.16b // ...............................*..................................... // eor v4.16B, v3.16B, v29.16B // ......................*.............................................. // aesr v10.16b, v20.16b // ...................*................................................. // eor v0.8B, v0.8B, v1.8B // .............................*....................................... // eor v29.16B, v2.16B, v9.16B // .................................*................................... // pmull v11.1q, v5.1d, v7.1d // ......*.............................................................. // aesr v10.16b, v21.16b // .....................*............................................... // eor v11.16B, v31.16B, v11.16B // .................*................................................... // pmull2 v31.1q, v1.2d, v12.2d // .........................................*........................... // aesr v10.16b, v22.16b // .......................*............................................. // eor v6.16B, v4.16B, v6.16B // ........................*............................................ // pmull v4.1q, v1.1d, v12.1d // ............................................*........................ // rev64 v5.16B, v8.16B // ..............................................*...................... // aesr v10.16b, v23.16b // .........................*........................................... // eor v11.16B, v6.16B, v11.16B // ..........................*.......................................... // aesr v10.16b, v24.16b // ...........................*......................................... // pmull v6.1q, v11.1d, v7.1d // .............................*....................................... // ldr q2, [x0, #-32] // ...........................*......................................... // str q8, [x2, #16] // ...........................................................*......... // aesr v10.16b, v25.16b // .................................*................................... // eor v30.16B, v30.16B, v6.16B // ................................*.................................... // aesr v10.16b, v26.16b // ...................................*................................. // rev64 v8.16B, v29.16B // ...................................*................................. // eor v6.16B, v2.16B, v28.16B // .....................................*............................... // aese v10.16b, v27.16b // .....................................*............................... // ext v11.16B, v11.16B, v11.16B, #8 // ............................*........................................ // str q29, [x2], #(4*16) // ...........................................................*......... // pmull2 v3.1q, v5.2d, v15.2d // .................................................*................... // eor v6.16B, v6.16B, v10.16B // .......................................*............................. // rev64 v29.16B, v6.16B // .........................................*........................... // eor v30.16B, v30.16B, v11.16B // ..................................*.................................. // ext v11.16B, v29.16B, v29.16B, #8 // ...........................................*......................... // ext v30.16B, v30.16B, v30.16B, #8 // ....................................*................................ // eor v2.16B, v11.16B, v29.16B // .............................................*....................... // pmull2 v10.1q, v29.2d, v13.2d // .............................................*....................... // eor v11.16B, v8.16B, v30.16B // ......................................*.............................. // str q6, [x2, #-32] // .........................................................*........... // mov d6, v5.d[1] // ................................................*.................... // pmull v0.1q, v0.1d, v14.1d // .......................................*............................. // eor v30.16B, v31.16B, v10.16B // ...............................................*..................... // pmull2 v10.1q, v2.2d, v14.2d // ...............................................*..................... // eor v6.8B, v6.8B, v5.8B // ..................................................*.................. // eor v10.16B, v0.16B, v10.16B // .................................................*................... // pmull v0.1q, v29.1d, v13.1d // ...........................................*......................... // pmull v2.1q, v6.1d, v17.1d // ....................................................*................ // eor v6.16B, v30.16B, v3.16B // ...................................................*................. // eor v3.16B, v10.16B, v2.16B // ......................................................*.............. // eor v0.16B, v4.16B, v0.16B // ..............................................*...................... // pmull v31.1q, v5.1d, v15.1d // ..................................................*.................. // ext v1.16B, v11.16B, v11.16B, #8 // ........................................*............................ // eor v30.16B, v0.16B, v31.16B // ....................................................*................ // pmull v31.1q, v11.1d, v16.1d // ................................................*.................... // eor v4.16B, v1.16B, v11.16B // ..........................................*.......................... // eor v30.16B, v30.16B, v31.16B // ......................................................*.............. // pmull2 v0.1q, v11.2d, v16.2d // ...................................................*................. // eor v5.16B, v6.16B, v0.16B // .....................................................*............... // ext v31.16B, v5.16B, v5.16B, #8 // .......................................................*............. // eor v6.16B, v30.16B, v5.16B // ........................................................*............ // pmull2 v29.1q, v4.2d, v17.2d // .....................................................*............... // eor v4.16B, v3.16B, v29.16B // ........................................................*............ // pmull v11.1q, v5.1d, v7.1d // .......................................................*............. // eor v11.16B, v31.16B, v11.16B // ..........................................................*.......... // eor v6.16B, v4.16B, v6.16B // ..........................................................*.......... // eor v11.16B, v6.16B, v11.16B // ............................................................*........ // pmull v6.1q, v11.1d, v7.1d // ..............................................................*...... // eor v30.16B, v30.16B, v6.16B // ................................................................*.... // ext v11.16B, v11.16B, v11.16B, #8 // ..............................................................*...... // eor v30.16B, v30.16B, v11.16B // ..................................................................*.. // ext v30.16B, v30.16B, v30.16B, #8 // ....................................................................* b Lloop_unrolled_start_end Lloop_unrolled_start_iter_1: ldr q29, [x0, #(3*16)] add w14, w13, #3 rev w14, w14 orr x14, x12, x14, lsl #32 stp x11, x14, [sp, #(STACK_BASE_AES_ST + 3*16)] // @slothy:writes=stack_3 ldr q0, [sp, #(STACK_BASE_AES_ST + 3*16)] // @slothy:reads=stack_3 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aese v0.16b, v27.16b eor v29.16b, v29.16b, v28.16b eor v0.16b, v29.16b, v0.16b str q0, [x2, #(3*16)] rev64 v0.16b, v0.16b // Low product pmull v8.1q, v0.1d, v12.1d // High product pmull2 v9.1q, v0.2d, v12.2d // Middle product mov d11, v0.d[1] eor v11.8b, v11.8b, v0.8b pmull v10.1q, v11.1d, v14.1d ldr q29, [x0, #(2*16)] add w14, w13, #2 rev w14, w14 orr x14, x12, x14, lsl #32 stp x11, x14, [sp, #(STACK_BASE_AES_ST + 2*16)] // @slothy:writes=stack_2 ldr q0, [sp, #(STACK_BASE_AES_ST + 2*16)] // @slothy:reads=stack_2 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aese v0.16b, v27.16b eor v29.16b, v29.16b, v28.16b eor v0.16b, v29.16b, v0.16b str q0, [x2, #(2*16)] rev64 v0.16b, v0.16b // Low product pmull v11.1q, v0.1d, v13.1d eor v8.16b, v8.16b, v11.16b // High product pmull2 v11.1q, v0.2d, v13.2d eor v9.16b, v9.16b, v11.16b // Middle product ext v11.16b, v0.16b, v0.16b, #8 eor v11.16b, v11.16b, v0.16b pmull2 v11.1q, v11.2d, v14.2d eor v10.16b, v10.16b, v11.16b ldr q29, [x0, #(1*16)] add w14, w13, #1 rev w14, w14 orr x14, x12, x14, lsl #32 stp x11, x14, [sp, #(STACK_BASE_AES_ST + 1*16)] // @slothy:writes=stack_1 ldr q0, [sp, #(STACK_BASE_AES_ST + 1*16)] // @slothy:reads=stack_1 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aese v0.16b, v27.16b eor v29.16b, v29.16b, v28.16b eor v0.16b, v29.16b, v0.16b str q0, [x2, #(1*16)] rev64 v0.16b, v0.16b // Low product pmull v11.1q, v0.1d, v15.1d eor v8.16b, v8.16b, v11.16b // High product pmull2 v11.1q, v0.2d, v15.2d eor v9.16b, v9.16b, v11.16b // Middle product mov d11, v0.d[1] eor v11.8b, v11.8b, v0.8b pmull v11.1q, v11.1d, v17.1d eor v10.16b, v10.16b, v11.16b ldr q29, [x0], #(4*16) add w14, w13, #0 rev w14, w14 orr x14, x12, x14, lsl #32 stp x11, x14, [sp, #(STACK_BASE_AES_ST + 0*16)] // @slothy:writes=stack_0 ldr q0, [sp, #(STACK_BASE_AES_ST + 0*16)] // @slothy:reads=stack_0 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aese v0.16b, v27.16b eor v29.16b, v29.16b, v28.16b eor v0.16b, v29.16b, v0.16b str q0, [x2], #(4*16) rev64 v0.16b, v0.16b eor v0.16b, v0.16b, v30.16b // Low product pmull v11.1q, v0.1d, v16.1d eor v8.16b, v8.16b, v11.16b // High product pmull2 v11.1q, v0.2d, v16.2d eor v9.16b, v9.16b, v11.16b // Middle product ext v11.16b, v0.16b, v0.16b, #8 eor v11.16b, v11.16b, v0.16b pmull2 v11.1q, v11.2d, v17.2d eor v10.16b, v10.16b, v11.16b eor v0.16b, v8.16b, v9.16b pmull v1.1q, v9.1d, v7.1d ext v9.16b, v9.16b, v9.16b, #8 eor v10.16b, v10.16b, v0.16b eor v1.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v1.16b pmull v9.1q, v10.1d, v7.1d eor v8.16b, v8.16b, v9.16b ext v10.16b, v10.16b, v10.16b, #8 eor v30.16b, v8.16b, v10.16b ext v30.16b, v30.16b, v30.16b, #8 add w13, w13, #UNROLL Lloop_unrolled_start_iter_1_end: Lloop_unrolled_start_end: Lloop_unrolled_end: cbz remainder, Lloop_1x_end Lloop_1x_start: ldr plain_q, [input], #16 aes_full_block aes_st, plain, res, 0 str res_q, [output], #16 ghash_init_with_tag_0 res, Ht1, Ht12, tag ghash_finalize tag add ctr, ctr, #1 sub remainder, remainder, #1 cbnz remainder, Lloop_1x_start Lloop_1x_end: // Return number of bytes processed mov x0, byte_len // Store new authentication tag rev64 tag.16b, tag.16b str tag_q, [tag_ptr] // Store updated counter // rev32 rtmp_ctr.16b, rtmp_ctr.16b // str rtmp_ctr_q, [ivec] rev ctr_tmp_w, ctr str ctr_tmp_w, [ivec, #12] restore_vregs restore_gprs Lenc_postamble_end: add sp, sp, #STACK_SIZE ret #endif #endif // !OPENSSL_NO_ASM && defined(__AARCH64EL__) && defined(__APPLE__) #if defined(__ELF__) // See https: // www.airs.com/blog/archives/518. .section .note.GNU-stack,"",%progbits #endif