// Copyright (c) 2022, ARM Inc. // Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. // Permission to use, copy, modify, and/or distribute this software for any // purpose with or without fee is hereby granted, provided that the above // copyright notice and this permission notice appear in all copies. // THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES // WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY // SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES // WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION // OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN // CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ // Author: Hanno Becker // This file was derived from the assembly generated from aesv8-gcm-armv8.pl, // written by Fangming Fang for the OpenSSL project, // and derived from https: // github.com/ARM-software/AArch64cryptolib, original // author Samuel Lee . // The code below is a 'clean' AArch64 implementation of AES-GCM emphasizing // the logic of the computation. It is meant as the input to manual audits / // formal verification, as well as automated micro-optimization such as done // by the SLOTHY superoptimizer (https: // github.com/slothy-optimizer/slothy). #if !defined(__has_feature) #define __has_feature(x) 0 #endif #if __has_feature(memory_sanitizer) && !defined(OPENSSL_NO_ASM) #define OPENSSL_NO_ASM #endif #include "aarch64-gcm-asm-compat.h" #if !defined(OPENSSL_NO_ASM) && defined(__AARCH64EL__) #if defined(__ELF__) .arch armv8-a+crypto .text .globl aes_gcm_dec_kernel_slothy_base_192 .hidden aes_gcm_dec_kernel_slothy_base_192 .type aes_gcm_dec_kernel_slothy_base_192,%function #elif defined(__APPLE__) .text .globl _aes_gcm_dec_kernel_slothy_base_192 .private_extern _aes_gcm_dec_kernel_slothy_base_192 #else #error Unknown configuration #endif #if __ARM_MAX_ARCH__ >= 8 // Arguments input .req x0 len_bits .req x1 output .req x2 tag_ptr .req x3 ivec .req x4 key .req x5 Htable .req x6 byte_len .req x15 constant_temp .req x25 count .req x1 full_blocks .req x7 remainder .req x9 unroll .req x10 ctr_tmp .req x14 ctr_tmp_w .req w14 ivec_0_63 .req x11 ivec_64_96 .req x12 ivec_64_96_w .req w12 ctr .req w13 ctr_x .req x13 aes_st .req v0 aes_st_q .req q0 aes_st_d .req d0 res .req v0 res_q .req q0 ghash_hi .req v9 ghash_lo .req v8 ghash_mid .req v10 ghash_mid_d .req d10 ghash_tmp .req v11 ghash_tmp_d .req d11 ghash_mod .req v7 ghash_mod_d .req d7 modulo_tmp0 .req v0 modulo_tmp1 .req v1 Ht1q .req q12 Ht2q .req q13 Ht12q .req q14 Ht1 .req v12 Ht2 .req v13 Ht12 .req v14 Ht3q .req Ht1q Ht4q .req Ht2q Ht34q .req Ht12q Ht3 .req Ht1 Ht4 .req Ht2 Ht34 .req Ht12 Ht5q .req Ht1q Ht6q .req Ht2q Ht56q .req Ht12q Ht5 .req Ht1 Ht6 .req Ht2 Ht56 .req Ht12 Ht7q .req Ht1q Ht8q .req Ht2q Ht78q .req Ht12q Ht7 .req Ht1 Ht8 .req Ht2 Ht78 .req Ht12 rk0q .req q18 rk1q .req q19 rk2q .req q20 rk3q .req q21 rk4q .req q22 rk5q .req q23 rk6q .req q24 rk7q .req q25 rk8q .req q26 rk9q .req q27 rk10q .req q28 rk11q .req q15 rk12q .req q16 rk0 .req v18 rk1 .req v19 rk2 .req v20 rk3 .req v21 rk4 .req v22 rk5 .req v23 rk6 .req v24 rk7 .req v25 rk8 .req v26 rk9 .req v27 rk10 .req v28 rk11 .req v15 rk12 .req v16 plain .req v29 plain_q .req q29 tag .req v30 tag_q .req q30 #define UNROLL 4 #define STACK_SIZE_GPRS (6*16) #define STACK_SIZE_VREGS (4*16) #define STACK_SIZE (STACK_SIZE_GPRS + STACK_SIZE_VREGS + UNROLL*16) #define STACK_BASE_GPRS (0) #define STACK_BASE_VREGS (STACK_SIZE_GPRS) #define STACK_BASE_AES_ST (STACK_SIZE_GPRS + STACK_SIZE_VREGS) /********************************************************************/ /* Generic preamble/postamble macros */ /********************************************************************/ .macro save_vregs stp d8, d9, [sp, #(STACK_BASE_VREGS + 16*0)] stp d10, d11, [sp, #(STACK_BASE_VREGS + 16*1)] stp d12, d13, [sp, #(STACK_BASE_VREGS + 16*2)] stp d14, d15, [sp, #(STACK_BASE_VREGS + 16*3)] .endm .macro restore_vregs ldp d8, d9, [sp, #(STACK_BASE_VREGS + 16*0)] ldp d10, d11, [sp, #(STACK_BASE_VREGS + 16*1)] ldp d12, d13, [sp, #(STACK_BASE_VREGS + 16*2)] ldp d14, d15, [sp, #(STACK_BASE_VREGS + 16*3)] .endm .macro save_gprs stp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] stp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] stp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] stp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] stp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] stp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] .endm .macro restore_gprs ldp x19, x20, [sp, #(STACK_BASE_GPRS + 16*0)] ldp x21, x22, [sp, #(STACK_BASE_GPRS + 16*1)] ldp x23, x24, [sp, #(STACK_BASE_GPRS + 16*2)] ldp x25, x26, [sp, #(STACK_BASE_GPRS + 16*3)] ldp x27, x28, [sp, #(STACK_BASE_GPRS + 16*4)] ldp x29, x30, [sp, #(STACK_BASE_GPRS + 16*5)] .endm // Derive number of iterations of unrolled loop and single-block loop .macro prepare_loop_counts mov unroll, #UNROLL // Number of AES Blocks (16b each) lsr full_blocks, byte_len, #4 // Number of iterations of the unrolled loop udiv count, full_blocks, unroll // Number of iterations for the tail loop handling 1 block each msub remainder, count, unroll, full_blocks .endm /********************************************************************/ /* AES related macros */ /********************************************************************/ .macro load_iv ldp ivec_0_63, ivec_64_96, [ivec] stp ivec_0_63, ivec_64_96, [sp, #(STACK_BASE_AES_ST + 0*16)] // @slothy:writes=stack_0 stp ivec_0_63, ivec_64_96, [sp, #(STACK_BASE_AES_ST + 1*16)] // @slothy:writes=stack_1 stp ivec_0_63, ivec_64_96, [sp, #(STACK_BASE_AES_ST + 2*16)] // @slothy:writes=stack_2 stp ivec_0_63, ivec_64_96, [sp, #(STACK_BASE_AES_ST + 3*16)] // @slothy:writes=stack_3 lsr ctr_x, ivec_64_96, #32 rev ctr, ctr orr ivec_64_96_w, ivec_64_96_w, ivec_64_96_w // clear top 32 bit .endm .macro next_ctr_init_aes aes_st, loc add ctr_tmp_w, ctr, #\loc rev ctr_tmp_w, ctr_tmp_w str ctr_tmp_w, [sp, #(STACK_BASE_AES_ST + \loc*16 + 12)] // @slothy:writes=stack_\loc ldr \aes_st\()_q, [sp, #(STACK_BASE_AES_ST + \loc*16)] // @slothy:reads=stack_\loc .endm // A single AES round // Prevent SLOTHY from unfolding because uArchs tend to fuse AESMC+AESE .macro aesr data, key // @slothy:no-unfold aese \data, \key aesmc \data, \data .endm .macro aesr_0_8 data, key aesr \data\().16b, \key\()0.16b aesr \data\().16b, \key\()1.16b aesr \data\().16b, \key\()2.16b aesr \data\().16b, \key\()3.16b aesr \data\().16b, \key\()4.16b aesr \data\().16b, \key\()5.16b aesr \data\().16b, \key\()6.16b aesr \data\().16b, \key\()7.16b aesr \data\().16b, \key\()8.16b .endm .macro aesr_9_10 data, key aesr \data\().16b, \key\()9.16b aesr \data\().16b, \key\()10.16b .endm .macro aesr_11_12 data, key aesr \data\().16b, \key\()11.16b aesr \data\().16b, \key\()12.16b .endm // Destructs inA .macro eor3 out, inA, inB, inC eor \inA, \inA, \inB eor \out, \inA, \inC .endm .macro aesr_final aes_st, plain, out aese \aes_st\().16b, rk11.16b eor3 \out\().16b, \aes_st\().16b, rk12.16b, \plain\().16b .endm .macro aes_full_block aes_st, input, output, loc next_ctr_init_aes \aes_st, \loc aesr_0_8 \aes_st\(), rk aesr_9_10 \aes_st\(), rk aesr_final \aes_st, \input, \output .endm .macro load_round_key i ldr rk\i\()q, [key, #((\i)*16)] .endm .macro load_round_keys load_round_key 0 load_round_key 1 load_round_key 2 load_round_key 3 load_round_key 4 load_round_key 5 load_round_key 6 load_round_key 7 load_round_key 8 load_round_key 9 load_round_key 10 load_round_key 11 load_round_key 12 .endm /********************************************************************/ /* Loading of H-table (precomputed H-powers for GHASH) */ /********************************************************************/ // This has to be synchronized with the H-table generation .macro load_h1 dst, dst_q ldr \dst_q, [Htable] .endm .macro load_h2 dst, dst_q ldr \dst_q, [Htable, #32] .endm .macro load_h3 dst, dst_q ldr \dst_q, [Htable, #48] .endm .macro load_h4 dst, dst_q ldr \dst_q, [Htable, #80] .endm .macro load_h5 dst, dst_q ldr \dst_q, [Htable, #96] .endm .macro load_h6 dst, dst_q ldr \dst_q, [Htable, #128] .endm .macro load_h7 dst, dst_q ldr \dst_q, [Htable, #144] .endm .macro load_h8 dst, dst_q ldr \dst_q, [Htable, #176] .endm .macro load_h12 dst, dst_q ldr \dst_q, [Htable, #16] .endm .macro load_h34 dst, dst_q ldr \dst_q, [Htable, #64] .endm .macro load_h56 dst, dst_q ldr \dst_q, [Htable, #112] .endm .macro load_h78 dst, dst_q ldr \dst_q, [Htable, #160] .endm .macro load_full_htable load_h1 Ht1, Ht1q load_h2 Ht2, Ht2q load_h3 Ht3, Ht3q load_h4 Ht4, Ht4q load_h5 Ht5, Ht5q load_h6 Ht6, Ht6q load_h12 Ht12, Ht12q load_h34 Ht34, Ht34q load_h56 Ht56, Ht56q .endm .macro load_htable_12 load_h1 Ht1, Ht1q load_h2 Ht2, Ht2q load_h12 Ht12, Ht12q .endm .macro load_htable_34 load_h3 Ht3, Ht3q load_h4 Ht4, Ht4q load_h34 Ht34, Ht34q .endm .macro load_htable_56 load_h5 Ht5, Ht5q load_h6 Ht6, Ht6q load_h56 Ht56, Ht56q .endm .macro load_htable_78 load_h7 Ht7, Ht7q load_h8 Ht8, Ht8q load_h78 Ht78, Ht78q .endm /********************************************************************/ /* Macros for GHASH udpate */ /********************************************************************/ .macro ghash_init_0 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_mid.1q, ghash_tmp.1d, \Hk_mid\().1d .endm .macro ghash_init_1 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_mid.1q, ghash_tmp.2d, \Hk_mid\().2d .endm .macro ghash_acc_0 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_tmp.1q, ghash_tmp.1d, \Hk_mid\().1d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_acc_1 input, Hk, Hk_mid rev64 \input\().16b, \input\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_tmp.1q, ghash_tmp.2d, \Hk_mid\().2d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_init_with_tag_0 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_mid.1q, ghash_tmp.1d, \Hk_mid\().1d .endm .macro ghash_init_with_tag_1 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_lo.1q, \input\().1d, \Hk\().1d // High product pmull2 ghash_hi.1q, \input\().2d, \Hk\().2d // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_mid.1q, ghash_tmp.2d, \Hk_mid\().2d .endm .macro ghash_acc_with_tag_0 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product mov ghash_tmp_d, \input\().d[1] eor ghash_tmp.8b, ghash_tmp.8b, \input\().8b pmull ghash_tmp.1q, ghash_tmp.1d, \Hk_mid\().1d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_acc_with_tag_1 input, Hk, Hk_mid, tag rev64 \input\().16b, \input\().16b eor \input\().16b, \input\().16b, \tag\().16b // Low product pmull ghash_tmp.1q, \input\().1d, \Hk\().1d eor ghash_lo.16b, ghash_lo.16b, ghash_tmp.16b // High product pmull2 ghash_tmp.1q, \input\().2d, \Hk\().2d eor ghash_hi.16b, ghash_hi.16b, ghash_tmp.16b // Middle product ext ghash_tmp.16b, \input\().16b, \input\().16b, #8 eor ghash_tmp.16b, ghash_tmp.16b, \input\().16b pmull2 ghash_tmp.1q, ghash_tmp.2d, \Hk_mid\().2d eor ghash_mid.16b, ghash_mid.16b, ghash_tmp.16b .endm .macro ghash_finalize tag eor modulo_tmp0.16b, ghash_lo.16b, ghash_hi.16b pmull modulo_tmp1.1q, ghash_hi.1d, ghash_mod.1d ext ghash_hi.16b, ghash_hi.16b, ghash_hi.16b, #8 eor ghash_mid.16b, ghash_mid.16b, modulo_tmp0.16b eor modulo_tmp1.16b, ghash_hi.16b, modulo_tmp1.16b eor ghash_mid.16b, ghash_mid.16b, modulo_tmp1.16b pmull ghash_hi.1q, ghash_mid.1d, ghash_mod.1d eor ghash_lo.16b, ghash_lo.16b, ghash_hi.16b ext ghash_mid.16b, ghash_mid.16b, ghash_mid.16b, #8 eor \tag\().16b, ghash_lo.16b, ghash_mid.16b ext \tag\().16b, \tag\().16b, \tag\().16b, #8 .endm .macro load_tag ldr tag_q, [tag_ptr] rev64 tag.16b, tag.16b .endm .macro prepare_ghash // Prepare constant for modular reduction movi ghash_mod.8b, #0xc2 shl ghash_mod_d, ghash_mod_d, #56 .endm /********************************************************************/ /* Core routine */ /********************************************************************/ .align 4 _aes_gcm_dec_kernel_slothy_base_192: aes_gcm_dec_kernel_slothy_base_192: #ifdef BORINGSSL_DISPATCH_TEST adrp x9,_BORINGSSL_function_hit@PAGE add x9, x9, _BORINGSSL_function_hit@PAGEOFF mov w10, #1 strb w10, [x9,#15] // kFlag_aes_gcm_slothy #endif AARCH64_VALID_CALL_TARGET sub sp, sp, #STACK_SIZE Ldec_preamble_start: save_gprs save_vregs lsr byte_len, len_bits, #3 load_round_keys load_tag load_iv prepare_loop_counts prepare_ghash Ldec_preamble_end: cbz count, Lloop_unrolled_end cmp count, #1 b.eq Lloop_unrolled_start_iter_1 // Instructions: 190 // Expected cycles: 101 // Expected IPC: 1.88 // // Cycle bound: 101.0 // IPC bound: 1.88 // // Wall time: 394.98s // User time: 394.98s // // ----------------------------------------- cycle (expected) -----------------------------------------> // 0 25 50 75 100 // |------------------------|------------------------|------------------------|------------------------| ldr q2, [x0, #32] // *.................................................................................................... add w25, w13, #0 // *.................................................................................................... add w28, w13, #1 // *.................................................................................................... add w7, w13, #2 // *.................................................................................................... ldr q4, [x0, #48] // .*................................................................................................... add w12, w13, #UNROLL // .*................................................................................................... add w11, w13, #3 // .*................................................................................................... rev w24, w28 // .*................................................................................................... ldr q1, [x0, #16] // ..*.................................................................................................. ldr q8, [x6, #32] // ..*.................................................................................................. rev w23, w7 // ..*.................................................................................................. add w13, w12, #UNROLL // ..*.................................................................................................. ldr q12, [x6] // ...*................................................................................................. rev w8, w11 // ...*................................................................................................. rev w19, w25 // ...*................................................................................................. str w23, [sp, #STACK_BASE_AES_ST + 44] // ...*................................................................................................. // @slothy:writes=stack_2 ldr q31, [x6, #48] // ....*................................................................................................ rev64 v29.16B, v2.16B // ....*................................................................................................ str w19, [sp, #STACK_BASE_AES_ST + 12] // ....*................................................................................................ // @slothy:writes=stack_0 add w29, w12, #0 // ....*................................................................................................ ldr q10, [sp, #STACK_BASE_AES_ST + 32] // .....*............................................................................................... // @slothy:reads=stack_2 rev64 v17.16B, v4.16B // .....*............................................................................................... rev w28, w29 // .....*............................................................................................... add w17, w12, #2 // .....*............................................................................................... pmull2 v11.1q, v29.2d, v8.2d // ......*.............................................................................................. rev64 v6.16B, v1.16B // ......*.............................................................................................. add w20, w12, #3 // ......*.............................................................................................. add w19, w12, #1 // ......*.............................................................................................. ldr q13, [sp, #STACK_BASE_AES_ST] // .......*............................................................................................. // @slothy:reads=stack_0 pmull2 v9.1q, v17.2d, v12.2d // .......*............................................................................................. rev w21, w19 // .......*............................................................................................. str w28, [sp, #STACK_BASE_AES_ST + 12] // .......*............................................................................................. // @slothy:writes=stack_0 mov d3, v6.d[1] // ........*............................................................................................ pmull v14.1q, v6.1d, v31.1d // ........*............................................................................................ str w24, [sp, #STACK_BASE_AES_ST + 28] // ........*............................................................................................ // @slothy:writes=stack_1 rev w10, w17 // ........*............................................................................................ pmull2 v31.1q, v6.2d, v31.2d // .........*........................................................................................... eor v0.16B, v9.16B, v11.16B // .........*........................................................................................... rev w26, w20 // .........*........................................................................................... str w10, [sp, #STACK_BASE_AES_ST + 44] // .........*........................................................................................... // @slothy:writes=stack_2 pmull v11.1q, v17.1d, v12.1d // ..........*.......................................................................................... ldr q5, [x6, #64] // ..........*.......................................................................................... eor v12.16B, v0.16B, v31.16B // ...........*......................................................................................... pmull v8.1q, v29.1d, v8.1d // ...........*......................................................................................... str w8, [sp, #STACK_BASE_AES_ST + 60] // ...........*......................................................................................... // @slothy:writes=stack_3 eor v3.8B, v3.8B, v6.8B // ............*........................................................................................ aesr v13.16b, v18.16b // ............*........................................................................................ aesr v10.16b, v18.16b // .............*....................................................................................... ldr q31, [sp, #STACK_BASE_AES_ST] // .............*....................................................................................... // @slothy:reads=stack_0 aesr v13.16b, v19.16b // ..............*...................................................................................... ldr q6, [x0], #(4*16) // ..............*...................................................................................... eor v11.16B, v11.16B, v8.16B // ...............*..................................................................................... pmull v0.1q, v3.1d, v5.1d // ...............*..................................................................................... ldr q9, [x6, #16] // ................*.................................................................................... aesr v13.16b, v20.16b // ................*.................................................................................... ldr q3, [sp, #STACK_BASE_AES_ST + 16] // .................*................................................................................... // @slothy:reads=stack_1 aesr v31.16b, v18.16b // .................*................................................................................... str w21, [sp, #STACK_BASE_AES_ST + 28] // .................*................................................................................... // @slothy:writes=stack_1 aesr v13.16b, v21.16b // ..................*.................................................................................. eor v8.16B, v11.16B, v14.16B // ..................*.................................................................................. aesr v31.16b, v19.16b // ...................*................................................................................. rev64 v11.16B, v6.16B // ...................*................................................................................. mov d14, v17.d[1] // ....................*................................................................................ aesr v13.16b, v22.16b // ....................*................................................................................ eor v11.16B, v11.16B, v30.16B // .....................*............................................................................... aesr v3.16b, v18.16b // .....................*............................................................................... eor v14.8B, v14.8B, v17.8B // ......................*.............................................................................. aesr v13.16b, v23.16b // ......................*.............................................................................. aesr v3.16b, v19.16b // .......................*............................................................................. ext v17.16B, v11.16B, v11.16B, #8 // .......................*............................................................................. aesr v13.16b, v24.16b // ........................*............................................................................ ldr q30, [x6, #80] // ........................*............................................................................ aesr v3.16b, v20.16b // .........................*........................................................................... eor v17.16B, v17.16B, v11.16B // .........................*........................................................................... aesr v13.16b, v25.16b // ..........................*.......................................................................... aesr v3.16b, v21.16b // ...........................*......................................................................... aesr v13.16b, v26.16b // ............................*........................................................................ aesr v3.16b, v22.16b // .............................*....................................................................... aesr v13.16b, v27.16b // ..............................*...................................................................... aesr v3.16b, v23.16b // ...............................*..................................................................... aesr v13.16b, v28.16b // ................................*.................................................................... aesr v3.16b, v24.16b // .................................*................................................................... aese v13.16b, v15.16b // ..................................*.................................................................. aesr v3.16b, v25.16b // ...................................*................................................................. pmull2 v17.1q, v17.2d, v5.2d // ....................................*................................................................ eor v5.16B, v13.16B, v16.16B // ....................................*................................................................ aesr v3.16b, v26.16b // .....................................*............................................................... ext v13.16B, v29.16B, v29.16B, #8 // .....................................*............................................................... eor v5.16B, v5.16B, v6.16B // ......................................*.............................................................. aesr v31.16b, v20.16b // ......................................*.............................................................. eor v6.16B, v13.16B, v29.16B // .......................................*............................................................. aesr v3.16b, v27.16b // .......................................*............................................................. str q5, [x2], #(4*16) // ........................................*............................................................ aesr v10.16b, v19.16b // ........................................*............................................................ ldr q13, [sp, #STACK_BASE_AES_ST + 16] // .........................................*........................................................... // @slothy:reads=stack_1 aesr v3.16b, v28.16b // .........................................*........................................................... aesr v10.16b, v20.16b // ..........................................*.......................................................... aese v3.16b, v15.16b // ...........................................*......................................................... aesr v10.16b, v21.16b // ............................................*........................................................ eor v5.16B, v3.16B, v16.16B // .............................................*....................................................... aesr v13.16b, v18.16b // .............................................*....................................................... ldr q3, [sp, #STACK_BASE_AES_ST + 32] // ..............................................*...................................................... // @slothy:reads=stack_2 aesr v10.16b, v22.16b // ..............................................*...................................................... eor v29.16B, v5.16B, v1.16B // ...............................................*..................................................... aesr v13.16b, v19.16b // ...............................................*..................................................... aesr v10.16b, v23.16b // ................................................*.................................................... str q29, [x2, #-48] // .................................................*................................................... aesr v13.16b, v20.16b // .................................................*................................................... aesr v10.16b, v24.16b // ..................................................*.................................................. aesr v13.16b, v21.16b // ...................................................*................................................. aesr v10.16b, v25.16b // ....................................................*................................................ aesr v13.16b, v22.16b // .....................................................*............................................... aesr v10.16b, v26.16b // ......................................................*.............................................. aesr v13.16b, v23.16b // .......................................................*............................................. aesr v10.16b, v27.16b // ........................................................*............................................ aesr v13.16b, v24.16b // .........................................................*........................................... aesr v10.16b, v28.16b // ..........................................................*.......................................... aesr v13.16b, v25.16b // ...........................................................*......................................... aese v10.16b, v15.16b // ............................................................*........................................ aesr v13.16b, v26.16b // .............................................................*....................................... aesr v31.16b, v21.16b // ..............................................................*...................................... eor v1.16B, v10.16B, v16.16B // ..............................................................*...................................... ldr q10, [x6] // ...............................................................*..................................... aesr v13.16b, v27.16b // ...............................................................*..................................... eor v29.16B, v1.16B, v2.16B // ................................................................*.................................... pmull2 v1.1q, v6.2d, v9.2d // ................................................................*.................................... aesr v13.16b, v28.16b // .................................................................*................................... ldr q2, [x0, #48] // .................................................................*................................... str q29, [x2, #-32] // ..................................................................*.................................. pmull2 v29.1q, v11.2d, v30.2d // ..................................................................*.................................. aese v13.16b, v15.16b // ...................................................................*................................. pmull v14.1q, v14.1d, v9.1d // ....................................................................*................................ eor v12.16B, v12.16B, v29.16B // ....................................................................*................................ pmull v5.1q, v11.1d, v30.1d // .....................................................................*............................... ldr q30, [sp, #STACK_BASE_AES_ST + 48] // .....................................................................*............................... // @slothy:reads=stack_3 aesr v3.16b, v18.16b // ......................................................................*.............................. ext v6.16B, v12.16B, v12.16B, #8 // ......................................................................*.............................. eor v9.16B, v8.16B, v5.16B // .......................................................................*............................. aesr v31.16b, v22.16b // .......................................................................*............................. eor v29.16B, v14.16B, v1.16B // ........................................................................*............................ aesr v3.16b, v19.16b // ........................................................................*............................ aesr v30.16b, v18.16b // .........................................................................*........................... eor v8.16B, v9.16B, v12.16B // .........................................................................*........................... eor v1.16B, v29.16B, v0.16B // ..........................................................................*.......................... aesr v3.16b, v20.16b // ..........................................................................*.......................... aesr v30.16b, v19.16b // ...........................................................................*......................... ldr q29, [x0, #32] // ...........................................................................*......................... eor v11.16B, v1.16B, v17.16B // ............................................................................*........................ aesr v3.16b, v21.16b // ............................................................................*........................ aesr v30.16b, v20.16b // .............................................................................*....................... ldr q17, [x0, #16] // .............................................................................*....................... eor v1.16B, v11.16B, v8.16B // ..............................................................................*...................... aesr v3.16b, v22.16b // ..............................................................................*...................... eor v13.16B, v13.16B, v16.16B // ...............................................................................*..................... aesr v30.16b, v21.16b // ...............................................................................*..................... aesr v3.16b, v23.16b // ................................................................................*.................... ldr q5, [x0], #(4*16) // ................................................................................*.................... aesr v30.16b, v22.16b // .................................................................................*................... rev64 v14.16B, v17.16B // .................................................................................*................... eor v11.16B, v13.16B, v17.16B // ..................................................................................*.................. aesr v3.16b, v24.16b // ..................................................................................*.................. aesr v30.16b, v23.16b // ...................................................................................*................. str q11, [x2, #16] // ....................................................................................*................ aesr v3.16b, v25.16b // ....................................................................................*................ pmull v0.1q, v12.1d, v7.1d // .....................................................................................*............... rev64 v11.16B, v5.16B // .....................................................................................*............... aesr v30.16b, v24.16b // ......................................................................................*.............. ldr q12, [x6, #32] // ......................................................................................*.............. eor v0.16B, v6.16B, v0.16B // .......................................................................................*............. aesr v3.16b, v26.16b // .......................................................................................*............. aesr v30.16b, v25.16b // ........................................................................................*............ aesr v3.16b, v27.16b // .........................................................................................*........... eor v8.16B, v1.16B, v0.16B // .........................................................................................*........... aesr v30.16b, v26.16b // ..........................................................................................*.......... aesr v3.16b, v28.16b // ...........................................................................................*......... ext v17.16B, v8.16B, v8.16B, #8 // ...........................................................................................*......... aesr v30.16b, v27.16b // ............................................................................................*........ aese v3.16b, v15.16b // .............................................................................................*....... aesr v30.16b, v28.16b // ..............................................................................................*...... aesr v31.16b, v23.16b // ...............................................................................................*..... eor v13.16B, v3.16B, v16.16B // ...............................................................................................*..... aese v30.16b, v15.16b // ................................................................................................*.... aesr v31.16b, v24.16b // .................................................................................................*... rev64 v3.16B, v2.16B // .................................................................................................*... eor v6.16B, v30.16B, v16.16B // ..................................................................................................*.. pmull v0.1q, v8.1d, v7.1d // ..................................................................................................*.. eor v1.16B, v13.16B, v29.16B // ...................................................................................................*. aesr v31.16b, v25.16b // ...................................................................................................*. eor v13.16B, v6.16B, v4.16B // ....................................................................................................* pmull v4.1q, v3.1d, v10.1d // ....................................................................................................* // ----------------------------------------- cycle (expected) -----------------------------------------> // 0 25 50 75 // |------------------------|------------------------|------------------------|------------------------- // add W, w13, #1 // *.................................................................................................... // rev W, W // .*................................................................................................... // add W, w13, #2 // *.................................................................................................... // str W, [sp, #STACK_BASE_AES_ST + 28] // ........*............................................................................................ // ldr Q, [sp, #STACK_BASE_AES_ST + 16] // .................*................................................................................... // add W, w13, #0 // *.................................................................................................... // add W, w13, #3 // .*................................................................................................... // rev W, W // ...*................................................................................................. // rev W, W // ..*.................................................................................................. // str W, [sp, #STACK_BASE_AES_ST + 12] // ....*................................................................................................ // add W, w13, #UNROLL // .*................................................................................................... // str W, [sp, #STACK_BASE_AES_ST + 44] // ...*................................................................................................. // aesr V.16b, v18.16b // .....................*............................................................................... // rev W, W // ...*................................................................................................. // aesr V.16b, v19.16b // .......................*............................................................................. // aesr V.16b, v20.16b // .........................*........................................................................... // aesr V.16b, v21.16b // ...........................*......................................................................... // aesr V.16b, v22.16b // .............................*....................................................................... // ldr Q, [sp, #STACK_BASE_AES_ST + 32] // .....*............................................................................................... // aesr V.16b, v23.16b // ...............................*..................................................................... // aesr V.16b, v18.16b // .............*....................................................................................... // ldr Q, [sp, #STACK_BASE_AES_ST] // .......*............................................................................................. // aesr V.16b, v24.16b // .................................*................................................................... // aesr V.16b, v25.16b // ...................................*................................................................. // aesr V.16b, v26.16b // .....................................*............................................................... // aesr V.16b, v27.16b // .......................................*............................................................. // aesr V.16b, v28.16b // .........................................*........................................................... // aese V.16b, v15.16b // ...........................................*......................................................... // aesr V.16b, v19.16b // ........................................*............................................................ // eor V.16B, V.16B, v16.16B // .............................................*....................................................... // aesr V.16b, v20.16b // ..........................................*.......................................................... // ldr Q, [x0, #16] // ..*.................................................................................................. // aesr V.16b, v18.16b // ............*........................................................................................ // ldr Q, [x0, #48] // .*................................................................................................... // aesr V.16b, v21.16b // ............................................*........................................................ // eor V.16B, V.16B, V.16B // ...............................................*..................................................... // aesr V.16b, v22.16b // ..............................................*...................................................... // rev64 V.16B, V.16B // ......*.............................................................................................. // str Q, [x2, #16] // .................................................*................................................... // aesr V.16b, v23.16b // ................................................*.................................................... // ldr Q, [x6] // ...*................................................................................................. // aesr V.16b, v24.16b // ..................................................*.................................................. // aesr V.16b, v19.16b // ..............*...................................................................................... // aesr V.16b, v25.16b // ....................................................*................................................ // aesr V.16b, v20.16b // ................*.................................................................................... // aesr V.16b, v26.16b // ......................................................*.............................................. // aesr V.16b, v21.16b // ..................*.................................................................................. // aesr V.16b, v27.16b // ........................................................*............................................ // ldr Q, [x6, #32] // ..*.................................................................................................. // aesr V.16b, v22.16b // ....................*................................................................................ // ldr Q, [x0], #(4*16) // ..............*...................................................................................... // aesr V.16b, v28.16b // ..........................................................*.......................................... // aesr V.16b, v23.16b // ......................*.............................................................................. // ldr Q, [x0, #-32] // *.................................................................................................... // aese V.16b, v15.16b // ............................................................*........................................ // rev64 V.16B, V.16B // .....*............................................................................................... // aesr V.16b, v24.16b // ........................*............................................................................ // eor V.16B, V.16B, v16.16B // ..............................................................*...................................... // rev64 V.16B, V.16B // ...................*................................................................................. // aesr V.16b, v25.16b // ..........................*.......................................................................... // eor V.16B, V.16B, V.16B // ................................................................*.................................... // pmull V.1q, V.1d, V.1d // ..........*.......................................................................................... // rev64 V.16B, V.16B // ....*................................................................................................ // aesr V.16b, v26.16b // ............................*........................................................................ // add W, W, #1 // ......*.............................................................................................. // pmull2 V.1q, V.2d, V.2d // .......*............................................................................................. // rev W, W // .......*............................................................................................. // str W, [sp, #STACK_BASE_AES_ST + 60] // ...........*......................................................................................... // ext V.16B, V.16B, V.16B, #8 // .....................................*............................................................... // aesr V.16b, v27.16b // ..............................*...................................................................... // add W, W, #2 // .....*............................................................................................... // str W, [sp, #STACK_BASE_AES_ST + 28] // .................*................................................................................... // pmull V.1q, V.1d, V.1d // ...........*......................................................................................... // ldr Q, [sp, #STACK_BASE_AES_ST + 48] // .....................................................................*............................... // ldr Q, [sp, #STACK_BASE_AES_ST + 16] // .........................................*........................................................... // aesr V.16b, v28.16b // ................................*.................................................................... // add W, W, #0 // ....*................................................................................................ // eor V.16B, V.16B, V.16B // ...............*..................................................................................... // pmull2 V.1q, V.2d, V.2d // ......*.............................................................................................. // add W, W, #3 // ......*.............................................................................................. // rev W, W // .....*............................................................................................... // eor V.16B, V.16B, V.16B // .......................................*............................................................. // aese V.16b, v15.16b // ..................................*.................................................................. // rev W, W // ........*............................................................................................ // str W, [sp, #STACK_BASE_AES_ST + 12] // .......*............................................................................................. // aesr V.16b, v18.16b // .........................................................................*........................... // add w13, W, #UNROLL // ..*.................................................................................................. // str W, [sp, #STACK_BASE_AES_ST + 44] // .........*........................................................................................... // eor V.16B, V.16B, v16.16B // ....................................*................................................................ // aesr V.16b, v18.16b // .............................................*....................................................... // rev w26, W // .........*........................................................................................... // aesr V.16b, v19.16b // ...........................................................................*......................... // eor V.16B, V.16B, V.16B // ......................................*.............................................................. // aesr V.16b, v19.16b // ...............................................*..................................................... // eor V.16B, V.16B, v30.16B // .....................*............................................................................... // aesr V.16b, v20.16b // .............................................................................*....................... // mov D, V.d[1] // ....................*................................................................................ // aesr V.16b, v20.16b // .................................................*................................................... // aesr V.16b, v21.16b // ...............................................................................*..................... // str Q, [x2, #32] // ..................................................................*.................................. // aesr V.16b, v21.16b // ...................................................*................................................. // ext V.16B, V.16B, V.16B, #8 // .......................*............................................................................. // aesr V.16b, v22.16b // .................................................................................*................... // ldr Q, [x6, #16] // ................*.................................................................................... // aesr V.16b, v22.16b // .....................................................*............................................... // ldr Q, [sp, #STACK_BASE_AES_ST + 32] // ..............................................*...................................................... // eor V.8B, V.8B, V.8B // ......................*.............................................................................. // aesr V.16b, v23.16b // ...................................................................................*................. // eor V.16B, V.16B, V.16B // .........................*........................................................................... // aesr V.16b, v23.16b // .......................................................*............................................. // str Q, [x2], #(4*16) // ........................................*............................................................ // aesr V.16b, v24.16b // ......................................................................................*.............. // aesr V.16b, v18.16b // ......................................................................*.............................. // ldr q31, [sp, #STACK_BASE_AES_ST] // .............*....................................................................................... // aesr V.16b, v25.16b // ........................................................................................*............ // pmull2 V.1q, V.2d, V.2d // ................................................................*.................................... // aesr V.16b, v26.16b // ..........................................................................................*.......... // aesr V.16b, v24.16b // .........................................................*........................................... // aesr V.16b, v27.16b // ............................................................................................*........ // mov D, V.d[1] // ........*............................................................................................ // aesr V.16b, v25.16b // ...........................................................*......................................... // aesr V.16b, v28.16b // ..............................................................................................*...... // eor V.16B, V.16B, V.16B // .........*........................................................................................... // aesr V.16b, v26.16b // .............................................................*....................................... // ldr Q, [x6, #48] // ....*................................................................................................ // aese V.16b, v15.16b // ................................................................................................*.... // ldr Q, [x6, #80] // ........................*............................................................................ // pmull V.1q, V.1d, V.1d // ....................................................................*................................ // ldr Q, [x6, #64] // ..........*.......................................................................................... // eor V.16B, V.16B, v16.16B // ..................................................................................................*.. // aesr V.16b, v27.16b // ...............................................................*..................................... // eor V.16B, V.16B, V.16B // ........................................................................*............................ // pmull V.1q, V.1d, V.1d // ........*............................................................................................ // aesr V.16b, v28.16b // .................................................................*................................... // eor v13.16B, V.16B, V.16B // ....................................................................................................* // eor V.16B, V.16B, V.16B // ..................*.................................................................................. // pmull2 V.1q, V.2d, V.2d // .........*........................................................................................... // aese V.16b, v15.16b // ...................................................................*................................. // eor V.8B, V.8B, V.8B // ............*........................................................................................ // aesr V.16b, v19.16b // ........................................................................*............................ // eor V.16B, V.16B, v16.16B // ...............................................................................*..................... // pmull V.1q, V.1d, V.1d // ...............*..................................................................................... // aesr V.16b, v20.16b // ..........................................................................*.......................... // ldr Q, [x0, #16] // .............................................................................*....................... // aesr v31.16b, v18.16b // .................*................................................................................... // ldr q2, [x0, #48] // .................................................................*................................... // aesr V.16b, v21.16b // ............................................................................*........................ // eor V.16B, V.16B, V.16B // ..........................................................................*.......................... // eor V.16B, V.16B, V.16B // ...........*......................................................................................... // pmull V.1q, V.1d, V.1d // .....................................................................*............................... // eor V.16B, V.16B, V.16B // ..................................................................................*.................. // aesr V.16b, v22.16b // ..............................................................................*...................... // pmull2 V.1q, V.2d, V.2d // ..................................................................*.................................. // rev64 v14.16B, V.16B // .................................................................................*................... // str Q, [x2, #16] // ....................................................................................*................ // aesr V.16b, v23.16b // ................................................................................*.................... // ldr q10, [x6] // ...............................................................*..................................... // pmull2 V.1q, V.2d, V.2d // ....................................*................................................................ // eor V.16B, V.16B, V.16B // ....................................................................*................................ // aesr V.16b, v24.16b // ..................................................................................*.................. // eor V.16B, V.16B, V.16B // ............................................................................*........................ // aesr v31.16b, v19.16b // ...................*................................................................................. // eor v9.16B, V.16B, V.16B // .......................................................................*............................. // pmull V.1q, V.1d, v7.1d // .....................................................................................*............... // ext V.16B, V.16B, V.16B, #8 // ......................................................................*.............................. // aesr V.16b, v25.16b // ....................................................................................*................ // eor V.16B, v9.16B, V.16B // .........................................................................*........................... // aesr v31.16b, v20.16b // ......................................*.............................................................. // eor V.16B, V.16B, V.16B // .......................................................................................*............. // aesr V.16b, v26.16b // .......................................................................................*............. // eor V.16B, V.16B, V.16B // ..............................................................................*...................... // aesr v31.16b, v21.16b // ..............................................................*...................................... // aesr V.16b, v27.16b // .........................................................................................*........... // ldr q12, [x6, #32] // ......................................................................................*.............. // eor V.16B, V.16B, V.16B // .........................................................................................*........... // aesr v31.16b, v22.16b // .......................................................................*............................. // ldr q5, [x0], #(4*16) // ................................................................................*.................... // aesr V.16b, v28.16b // ...........................................................................................*......... // ext v17.16B, V.16B, V.16B, #8 // ...........................................................................................*......... // aesr v31.16b, v23.16b // ...............................................................................................*..... // ldr q29, [x0, #-32] // ...........................................................................*......................... // aese V.16b, v15.16b // .............................................................................................*....... // rev64 v3.16B, v2.16B // .................................................................................................*... // aesr v31.16b, v24.16b // .................................................................................................*... // pmull v0.1q, V.1d, v7.1d // ..................................................................................................*.. // eor V.16B, V.16B, v16.16B // ...............................................................................................*..... // rev64 v11.16B, v5.16B // .....................................................................................*............... // aesr v31.16b, v25.16b // ...................................................................................................*. // eor v1.16B, V.16B, v29.16B // ...................................................................................................*. // pmull v4.1q, v3.1d, v10.1d // ....................................................................................................* sub count, count, #2 cbz count, Lloop_unrolled_start_postamble Lloop_unrolled_start: // Instructions: 132 // Expected cycles: 62 // Expected IPC: 2.13 // // Cycle bound: 62.0 // IPC bound: 2.13 // // Wall time: 2464.01s // User time: 2464.01s // // --------------------- cycle (expected) ----------------------> // 0 25 50 // |------------------------|------------------------|----------- rev64 v30.16B, v29.16B // *............................................................. aesr v31.16b, v26.16b // *............................................................. add w25, w13, #1 // e............................................................. pmull2 v10.1q, v3.2d, v10.2d // .*............................................................ eor v0.16B, v9.16B, v0.16B // .l............................................................ rev w8, w25 // .e............................................................ str w26, [sp, #STACK_BASE_AES_ST + 60] // .*............................................................ // @slothy:writes=stack_3 ext v29.16B, v30.16B, v30.16B, #8 // ..*........................................................... aesr v31.16b, v27.16b // ..*........................................................... add w27, w13, #2 // ..e........................................................... str w8, [sp, #STACK_BASE_AES_ST + 28] // ..e........................................................... // @slothy:writes=stack_1 pmull v9.1q, v30.1d, v12.1d // ...*.......................................................... ldr q8, [sp, #STACK_BASE_AES_ST + 48] // ...*.......................................................... // @slothy:reads=stack_3 ldr q6, [sp, #STACK_BASE_AES_ST + 16] // ....e......................................................... // @slothy:reads=stack_1 aesr v31.16b, v28.16b // ....*......................................................... add w24, w13, #0 // ....e......................................................... eor v9.16B, v4.16B, v9.16B // .....*........................................................ pmull2 v4.1q, v30.2d, v12.2d // .....*........................................................ add w11, w13, #3 // .....e........................................................ rev w25, w24 // .....e........................................................ eor v12.16B, v29.16B, v30.16B // ......*....................................................... aese v31.16b, v15.16b // ......*....................................................... rev w21, w27 // ......e....................................................... str w25, [sp, #STACK_BASE_AES_ST + 12] // ......e....................................................... // @slothy:writes=stack_0 eor v29.16B, v0.16B, v17.16B // .......l...................................................... aesr v8.16b, v18.16b // .......*...................................................... add w13, w13, #UNROLL // .......e...................................................... str w21, [sp, #STACK_BASE_AES_ST + 44] // .......e...................................................... // @slothy:writes=stack_2 eor v17.16B, v31.16B, v16.16B // ........*..................................................... aesr v6.16b, v18.16b // ........e..................................................... rev w26, w11 // ........e..................................................... ext v30.16B, v29.16B, v29.16B, #8 // .........l.................................................... aesr v8.16b, v19.16b // .........*.................................................... eor v29.16B, v17.16B, v5.16B // ..........*................................................... aesr v6.16b, v19.16b // ..........e................................................... eor v11.16B, v11.16B, v30.16B // ...........*.................................................. aesr v8.16b, v20.16b // ...........*.................................................. mov d31, v3.d[1] // ............*................................................. aesr v6.16b, v20.16b // ............e................................................. aesr v8.16b, v21.16b // .............*................................................ str q1, [x2, #32] // .............*................................................ aesr v6.16b, v21.16b // ..............e............................................... ext v17.16B, v11.16B, v11.16B, #8 // ..............*............................................... aesr v8.16b, v22.16b // ...............*.............................................. ldr q30, [x6, #16] // ...............*.............................................. aesr v6.16b, v22.16b // ................e............................................. ldr q1, [sp, #STACK_BASE_AES_ST + 32] // ................e............................................. // @slothy:reads=stack_2 eor v0.8B, v31.8B, v3.8B // .................*............................................ aesr v8.16b, v23.16b // .................*............................................ eor v17.16B, v17.16B, v11.16B // ..................*........................................... aesr v6.16b, v23.16b // ..................e........................................... str q29, [x2], #(4*16) // ...................*.......................................... aesr v8.16b, v24.16b // ...................*.......................................... aesr v1.16b, v18.16b // ....................e......................................... ldr q31, [sp, #STACK_BASE_AES_ST] // .....................e........................................ // @slothy:reads=stack_0 aesr v8.16b, v25.16b // .....................*........................................ str q13, [x2, #-80] // ......................l....................................... pmull2 v3.1q, v12.2d, v30.2d // ......................*....................................... aesr v8.16b, v26.16b // .......................*...................................... aesr v6.16b, v24.16b // ........................e..................................... aesr v8.16b, v27.16b // .........................*.................................... mov d29, v14.d[1] // ..........................*................................... aesr v6.16b, v25.16b // ..........................e................................... aesr v8.16b, v28.16b // ...........................*.................................. eor v5.16B, v10.16B, v4.16B // ...........................*.................................. aesr v6.16b, v26.16b // ............................e................................. ldr q4, [x6, #48] // ............................*................................. aese v8.16b, v15.16b // .............................*................................ ldr q10, [x6, #80] // .............................*................................ pmull v30.1q, v0.1d, v30.1d // ..............................*............................... ldr q0, [x6, #64] // ..............................*............................... eor v8.16B, v8.16B, v16.16B // ...............................*.............................. aesr v6.16b, v27.16b // ...............................e.............................. eor v12.16B, v30.16B, v3.16B // ................................*............................. pmull v3.1q, v14.1d, v4.1d // ................................*............................. aesr v6.16b, v28.16b // .................................e............................ eor v13.16B, v8.16B, v2.16B // .................................*............................ eor v3.16B, v9.16B, v3.16B // ..................................*........................... pmull2 v8.1q, v14.2d, v4.2d // ..................................*........................... aese v6.16b, v15.16b // ...................................e.......................... eor v9.8B, v29.8B, v14.8B // ...................................*.......................... aesr v1.16b, v19.16b // ....................................e......................... eor v4.16B, v6.16B, v16.16B // .....................................e........................ pmull v30.1q, v9.1d, v0.1d // .....................................*........................ aesr v1.16b, v20.16b // ......................................e....................... ldr q6, [x0, #16] // ......................................e....................... aesr v31.16b, v18.16b // .......................................e...................... ldr q2, [x0, #48] // .......................................e...................... aesr v1.16b, v21.16b // ........................................e..................... eor v29.16B, v12.16B, v30.16B // ........................................*..................... eor v30.16B, v5.16B, v8.16B // .........................................*.................... pmull v8.1q, v11.1d, v10.1d // .........................................*.................... eor v4.16B, v4.16B, v6.16B // ..........................................e................... aesr v1.16b, v22.16b // ..........................................e................... pmull2 v5.1q, v11.2d, v10.2d // ...........................................*.................. rev64 v14.16B, v6.16B // ...........................................e.................. str q4, [x2, #16] // ............................................e................. aesr v1.16b, v23.16b // ............................................e................. ldr q10, [x6] // .............................................e................ pmull2 v17.1q, v17.2d, v0.2d // .............................................*................ eor v4.16B, v30.16B, v5.16B // ..............................................*............... aesr v1.16b, v24.16b // ..............................................e............... eor v12.16B, v29.16B, v17.16B // ...............................................*.............. aesr v31.16b, v19.16b // ...............................................e.............. eor v9.16B, v3.16B, v8.16B // ................................................*............. pmull v6.1q, v4.1d, v7.1d // ................................................*............. ext v17.16B, v4.16B, v4.16B, #8 // .................................................*............ aesr v1.16b, v25.16b // .................................................e............ eor v0.16B, v9.16B, v4.16B // ..................................................*........... aesr v31.16b, v20.16b // ..................................................e........... eor v11.16B, v17.16B, v6.16B // ...................................................*.......... aesr v1.16b, v26.16b // ...................................................e.......... eor v3.16B, v12.16B, v0.16B // ....................................................*......... aesr v31.16b, v21.16b // ....................................................e......... aesr v1.16b, v27.16b // .....................................................e........ ldr q12, [x6, #32] // .....................................................e........ eor v6.16B, v3.16B, v11.16B // ......................................................*....... aesr v31.16b, v22.16b // ......................................................e....... ldr q5, [x0], #(4*16) // .......................................................e...... aesr v1.16b, v28.16b // .......................................................e...... ext v17.16B, v6.16B, v6.16B, #8 // ........................................................*..... aesr v31.16b, v23.16b // ........................................................e..... ldr q29, [x0, #-32] // .........................................................e.... aese v1.16b, v15.16b // .........................................................e.... rev64 v3.16B, v2.16B // ..........................................................e... aesr v31.16b, v24.16b // ..........................................................e... pmull v0.1q, v6.1d, v7.1d // ...........................................................*.. eor v30.16B, v1.16B, v16.16B // ...........................................................e.. rev64 v11.16B, v5.16B // ............................................................e. aesr v31.16b, v25.16b // ............................................................e. eor v1.16B, v30.16B, v29.16B // .............................................................e pmull v4.1q, v3.1d, v10.1d // .............................................................e // ---------------------------------------------------------------- cycle (expected) ----------------------------------------------------------------> // 0 25 50 75 100 125 // |------------------------|------------------------|------------------------|------------------------|------------------------|--------------------- // ldr q12, [x6] // .............................................e................'............................................~................'...................... // ldr q13, [x6, #32] // .....................................................e........'....................................................~........'...................... // ldr q14, [x6, #16] // ...............~..............................................'..............*..............................................'..............~....... // ldr q29, [x0, #(3*16)] // .......................................e......................'......................................~......................'...................... // add w14, w13, #3 // .....e........................................................'....~........................................................'....~................. // rev w14, w14 // ........e.....................................................'.......~.....................................................'.......~.............. // str w14, [sp, #(STACK_BASE_AES_ST + 3*16 + 12)] // .~............................................................'*............................................................'~..................... // ldr q0, [sp, #(STACK_BASE_AES_ST + 3*16)] // ...~..........................................................'..*..........................................................'..~................... // aesr v0.16b, v18.16b // .......~......................................................'......*......................................................'......~............... // aesr v0.16b, v19.16b // .........~....................................................'........*....................................................'........~............. // aesr v0.16b, v20.16b // ...........~..................................................'..........*..................................................'..........~........... // aesr v0.16b, v21.16b // .............~................................................'............*................................................'............~......... // aesr v0.16b, v22.16b // ...............~..............................................'..............*..............................................'..............~....... // aesr v0.16b, v23.16b // .................~............................................'................*............................................'................~..... // aesr v0.16b, v24.16b // ...................~..........................................'..................*..........................................'..................~... // aesr v0.16b, v25.16b // .....................~........................................'....................*........................................'....................~. // aesr v0.16b, v26.16b // .......................~......................................'......................*......................................'...................... // aesr v0.16b, v27.16b // .........................~....................................'........................*....................................'...................... // aesr v0.16b, v28.16b // ...........................~..................................'..........................*..................................'...................... // aese v0.16b, v15.16b // .............................~................................'............................*................................'...................... // eor v0.16b, v0.16b, v16.16b // ...............................~..............................'..............................*..............................'...................... // eor v0.16b, v0.16b, v29.16b // .................................~............................'................................*............................'...................... // str q0, [x2, #(3*16)] // ......................~.......................................'.....................~.......................................'.....................l // rev64 v29.16b, v29.16b // ..........................................................e...'.........................................................~...'...................... // pmull v8.1q, v29.1d, v12.1d // .............................................................e'............................................................~'...................... // pmull2 v9.1q, v29.2d, v12.2d // .~............................................................'*............................................................'~..................... // mov d11, v29.d[1] // ............~.................................................'...........*.................................................'...........~.......... // eor v11.8b, v11.8b, v29.8b // .................~............................................'................*............................................'................~..... // pmull v10.1q, v11.1d, v14.1d // ..............................~...............................'.............................*...............................'...................... // ldr q29, [x0, #(2*16)] // .........................................................e....'........................................................~....'...................... // add w14, w13, #2 // ..e...........................................................'.~...........................................................'.~.................... // rev w14, w14 // ......e.......................................................'.....~.......................................................'.....~................ // str w14, [sp, #(STACK_BASE_AES_ST + 2*16 + 12)] // .......e......................................................'......~......................................................'......~............... // ldr q0, [sp, #(STACK_BASE_AES_ST + 2*16)] // ................e.............................................'...............~.............................................'...............~...... // aesr v0.16b, v18.16b // ....................e.........................................'...................~.........................................'...................~.. // aesr v0.16b, v19.16b // ....................................e.........................'...................................~.........................'...................... // aesr v0.16b, v20.16b // ......................................e.......................'.....................................~.......................'...................... // aesr v0.16b, v21.16b // ........................................e.....................'.......................................~.....................'...................... // aesr v0.16b, v22.16b // ..........................................e...................'.........................................~...................'...................... // aesr v0.16b, v23.16b // ............................................e.................'...........................................~.................'...................... // aesr v0.16b, v24.16b // ..............................................e...............'.............................................~...............'...................... // aesr v0.16b, v25.16b // .................................................e............'................................................~............'...................... // aesr v0.16b, v26.16b // ...................................................e..........'..................................................~..........'...................... // aesr v0.16b, v27.16b // .....................................................e........'....................................................~........'...................... // aesr v0.16b, v28.16b // .......................................................e......'......................................................~......'...................... // aese v0.16b, v15.16b // .........................................................e....'........................................................~....'...................... // eor v0.16b, v0.16b, v16.16b // ...........................................................e..'..........................................................~..'...................... // eor v0.16b, v0.16b, v29.16b // .............................................................e'............................................................~'...................... // str q0, [x2, #(2*16)] // .............~................................................'............*................................................'............~......... // rev64 v29.16b, v29.16b // ~.............................................................*.............................................................~...................... // pmull v11.1q, v29.1d, v13.1d // ...~..........................................................'..*..........................................................'..~................... // eor v8.16b, v8.16b, v11.16b // .....~........................................................'....*........................................................'....~................. // pmull2 v11.1q, v29.2d, v13.2d // .....~........................................................'....*........................................................'....~................. // eor v9.16b, v9.16b, v11.16b // ...........................~..................................'..........................*..................................'...................... // ext v11.16b, v29.16b, v29.16b, #8 // ..~...........................................................'.*...........................................................'.~.................... // eor v11.16b, v11.16b, v29.16b // ......~.......................................................'.....*.......................................................'.....~................ // pmull2 v11.1q, v11.2d, v14.2d // ......................~.......................................'.....................*.......................................'...................... // eor v10.16b, v10.16b, v11.16b // ................................~.............................'...............................*.............................'...................... // ldr q12, [x6, #48] // ............................~.................................'...........................*.................................'...................... // ldr q13, [x6, #80] // .............................~................................'............................*................................'...................... // ldr q14, [x6, #64] // ..............................~...............................'.............................*...............................'...................... // ldr q29, [x0, #(1*16)] // ......................................e.......................'.....................................~.......................'...................... // add w14, w13, #1 // e.............................................................~.............................................................~...................... // rev w14, w14 // .e............................................................'~............................................................'~..................... // str w14, [sp, #(STACK_BASE_AES_ST + 1*16 + 12)] // ..e...........................................................'.~...........................................................'.~.................... // ldr q0, [sp, #(STACK_BASE_AES_ST + 1*16)] // ....e.........................................................'...~.........................................................'...~.................. // aesr v0.16b, v18.16b // ........e.....................................................'.......~.....................................................'.......~.............. // aesr v0.16b, v19.16b // ..........e...................................................'.........~...................................................'.........~............ // aesr v0.16b, v20.16b // ............e.................................................'...........~.................................................'...........~.......... // aesr v0.16b, v21.16b // ..............e...............................................'.............~...............................................'.............~........ // aesr v0.16b, v22.16b // ................e.............................................'...............~.............................................'...............~...... // aesr v0.16b, v23.16b // ..................e...........................................'.................~...........................................'.................~.... // aesr v0.16b, v24.16b // ........................e.....................................'.......................~.....................................'...................... // aesr v0.16b, v25.16b // ..........................e...................................'.........................~...................................'...................... // aesr v0.16b, v26.16b // ............................e.................................'...........................~.................................'...................... // aesr v0.16b, v27.16b // ...............................e..............................'..............................~..............................'...................... // aesr v0.16b, v28.16b // .................................e............................'................................~............................'...................... // aese v0.16b, v15.16b // ...................................e..........................'..................................~..........................'...................... // eor v0.16b, v0.16b, v16.16b // .....................................e........................'....................................~........................'...................... // eor v0.16b, v0.16b, v29.16b // ..........................................e...................'.........................................~...................'...................... // str q0, [x2, #(1*16)] // ............................................e.................'...........................................~.................'...................... // rev64 v29.16b, v29.16b // ...........................................e..................'..........................................~..................'...................... // pmull v11.1q, v29.1d, v12.1d // ................................~.............................'...............................*.............................'...................... // eor v8.16b, v8.16b, v11.16b // ..................................~...........................'.................................*...........................'...................... // pmull2 v11.1q, v29.2d, v12.2d // ..................................~...........................'.................................*...........................'...................... // eor v9.16b, v9.16b, v11.16b // .........................................~....................'........................................*....................'...................... // mov d11, v29.d[1] // ..........................~...................................'.........................*...................................'...................... // eor v11.8b, v11.8b, v29.8b // ...................................~..........................'..................................*..........................'...................... // pmull v11.1q, v11.1d, v14.1d // .....................................~........................'....................................*........................'...................... // eor v10.16b, v10.16b, v11.16b // ........................................~.....................'.......................................*.....................'...................... // ldr q29, [x0], #(4*16) // .......................................................e......'......................................................~......'...................... // add w14, w13, #0 // ....e.........................................................'...~.........................................................'...~.................. // rev w14, w14 // .....e........................................................'....~........................................................'....~................. // str w14, [sp, #(STACK_BASE_AES_ST + 0*16 + 12)] // ......e.......................................................'.....~.......................................................'.....~................ // ldr q0, [sp, #(STACK_BASE_AES_ST + 0*16)] // .....................e........................................'....................~........................................'....................~. // aesr v0.16b, v18.16b // .......................................e......................'......................................~......................'...................... // aesr v0.16b, v19.16b // ...............................................e..............'..............................................~..............'...................... // aesr v0.16b, v20.16b // ..................................................e...........'.................................................~...........'...................... // aesr v0.16b, v21.16b // ....................................................e.........'...................................................~.........'...................... // aesr v0.16b, v22.16b // ......................................................e.......'.....................................................~.......'...................... // aesr v0.16b, v23.16b // ........................................................e.....'.......................................................~.....'...................... // aesr v0.16b, v24.16b // ..........................................................e...'.........................................................~...'...................... // aesr v0.16b, v25.16b // ............................................................e.'...........................................................~.'...................... // aesr v0.16b, v26.16b // ~.............................................................*.............................................................~...................... // aesr v0.16b, v27.16b // ..~...........................................................'.*...........................................................'.~.................... // aesr v0.16b, v28.16b // ....~.........................................................'...*.........................................................'...~.................. // aese v0.16b, v15.16b // ......~.......................................................'.....*.......................................................'.....~................ // eor v0.16b, v0.16b, v16.16b // ........~.....................................................'.......*.....................................................'.......~.............. // eor v0.16b, v0.16b, v29.16b // ..........~...................................................'.........*...................................................'.........~............ // str q0, [x2], #(4*16) // ...................~..........................................'..................*..........................................'..................~... // rev64 v29.16b, v29.16b // ............................................................e.'...........................................................~.'...................... // eor v29.16b, v29.16b, v30.16b // ...........~..................................................'..........*..................................................'..........~........... // pmull v11.1q, v29.1d, v13.1d // .........................................~....................'........................................*....................'...................... // eor v8.16b, v8.16b, v11.16b // ................................................~.............'...............................................*.............'...................... // pmull2 v11.1q, v29.2d, v13.2d // ...........................................~..................'..........................................*..................'...................... // eor v9.16b, v9.16b, v11.16b // ..............................................~...............'.............................................*...............'...................... // ext v11.16b, v29.16b, v29.16b, #8 // ..............~...............................................'.............*...............................................'.............~........ // eor v11.16b, v11.16b, v29.16b // ..................~...........................................'.................*...........................................'.................~.... // pmull2 v11.1q, v11.2d, v14.2d // .............................................~................'............................................*................'...................... // eor v10.16b, v10.16b, v11.16b // ...............................................~..............'..............................................*..............'...................... // eor v0.16b, v8.16b, v9.16b // ..................................................~...........'.................................................*...........'...................... // pmull v1.1q, v9.1d, v7.1d // ................................................~.............'...............................................*.............'...................... // ext v9.16b, v9.16b, v9.16b, #8 // .................................................~............'................................................*............'...................... // eor v10.16b, v10.16b, v0.16b // ....................................................~.........'...................................................*.........'...................... // eor v1.16b, v9.16b, v1.16b // ...................................................~..........'..................................................*..........'...................... // eor v10.16b, v10.16b, v1.16b // ......................................................~.......'.....................................................*.......'...................... // pmull v9.1q, v10.1d, v7.1d // ...........................................................~..'..........................................................*..'...................... // eor v8.16b, v8.16b, v9.16b // .~............................................................'~............................................................'l..................... // ext v10.16b, v10.16b, v10.16b, #8 // ........................................................~.....'.......................................................*.....'...................... // eor v30.16b, v8.16b, v10.16b // .......~......................................................'......~......................................................'......l............... // ext v30.16b, v30.16b, v30.16b, #8 // .........~....................................................'........~....................................................'........l............. // add w13, w13, #UNROLL // .......e......................................................'......~......................................................'......~............... sub count, count, #1 cbnz count, Lloop_unrolled_start Lloop_unrolled_start_postamble:// end of loop kernel // Instructions: 74 // Expected cycles: 38 // Expected IPC: 1.95 // // Cycle bound: 38.0 // IPC bound: 1.95 // // Wall time: 8.16s // User time: 8.16s // // --------- cycle (expected) ----------> // 0 25 // |------------------------|------------ ldr q30, [x6, #16] // *..................................... mov d6, v3.d[1] // *..................................... str w26, [sp, #STACK_BASE_AES_ST + 60] // *..................................... // @slothy:writes=stack_3 pmull2 v8.1q, v3.2d, v10.2d // .*.................................... eor v9.16B, v9.16B, v0.16B // .*.................................... ldr q0, [sp, #STACK_BASE_AES_ST + 48] // ..*................................... // @slothy:reads=stack_3 eor v10.8B, v6.8B, v3.8B // ..*................................... eor v9.16B, v9.16B, v17.16B // ...*.................................. rev64 v29.16B, v29.16B // ...*.................................. pmull v3.1q, v10.1d, v30.1d // ....*................................. ldr q6, [x6, #80] // ....*................................. pmull2 v10.1q, v29.2d, v12.2d // .....*................................ ldr q17, [x6, #48] // .....*................................ ext v9.16B, v9.16B, v9.16B, #8 // ......*............................... aesr v0.16b, v18.16b // ......*............................... aesr v31.16b, v26.16b // .......*.............................. eor v10.16B, v8.16B, v10.16B // .......*.............................. eor v9.16B, v11.16B, v9.16B // ........*............................. aesr v0.16b, v19.16b // ........*............................. pmull v11.1q, v29.1d, v12.1d // .........*............................ ext v12.16B, v29.16B, v29.16B, #8 // .........*............................ aesr v0.16b, v20.16b // ..........*........................... ext v8.16B, v9.16B, v9.16B, #8 // ..........*........................... eor v12.16B, v12.16B, v29.16B // ...........*.......................... pmull2 v29.1q, v14.2d, v17.2d // ...........*.......................... eor v4.16B, v4.16B, v11.16B // ............*......................... aesr v0.16b, v21.16b // ............*......................... pmull2 v30.1q, v12.2d, v30.2d // .............*........................ eor v29.16B, v10.16B, v29.16B // .............*........................ aesr v0.16b, v22.16b // ..............*....................... mov d10, v14.d[1] // ..............*....................... eor v8.16B, v8.16B, v9.16B // ...............*...................... pmull2 v11.1q, v9.2d, v6.2d // ...............*...................... aesr v0.16b, v23.16b // ................*..................... ldr q12, [x6, #64] // ................*..................... eor v29.16B, v29.16B, v11.16B // .................*.................... pmull v11.1q, v14.1d, v17.1d // .................*.................... aesr v0.16b, v24.16b // ..................*................... eor v10.8B, v10.8B, v14.8B // ..................*................... pmull v17.1q, v9.1d, v6.1d // ...................*.................. eor v14.16B, v3.16B, v30.16B // ...................*.................. aesr v0.16b, v25.16b // ....................*................. eor v6.16B, v4.16B, v11.16B // ....................*................. ext v30.16B, v29.16B, v29.16B, #8 // .....................*................ pmull v9.1q, v10.1d, v12.1d // .....................*................ pmull2 v11.1q, v8.2d, v12.2d // ......................*............... eor v12.16B, v6.16B, v17.16B // ......................*............... eor v3.16B, v14.16B, v9.16B // .......................*.............. aesr v0.16b, v26.16b // .......................*.............. eor v17.16B, v12.16B, v29.16B // ........................*............. pmull v6.1q, v29.1d, v7.1d // ........................*............. aesr v0.16b, v27.16b // .........................*............ eor v9.16B, v3.16B, v11.16B // .........................*............ eor v8.16B, v30.16B, v6.16B // ..........................*........... aesr v31.16b, v27.16b // ..........................*........... aesr v0.16b, v28.16b // ...........................*.......... eor v9.16B, v9.16B, v17.16B // ...........................*.......... str q13, [x2, #-16] // ............................*......... aesr v31.16b, v28.16b // ............................*......... eor v4.16B, v9.16B, v8.16B // .............................*........ aese v0.16b, v15.16b // .............................*........ str q1, [x2, #32] // ..............................*....... aese v31.16b, v15.16b // ..............................*....... eor v9.16B, v0.16B, v16.16B // ...............................*...... pmull v14.1q, v4.1d, v7.1d // ...............................*...... ext v6.16B, v4.16B, v4.16B, #8 // ................................*..... eor v3.16B, v31.16B, v16.16B // ................................*..... eor v12.16B, v12.16B, v14.16B // .................................*.... eor v13.16B, v9.16B, v2.16B // .................................*.... eor v31.16B, v3.16B, v5.16B // ..................................*... str q13, [x2, #48] // ...................................*.. eor v14.16B, v12.16B, v6.16B // ...................................*.. str q31, [x2], #(4*16) // ....................................*. ext v30.16B, v14.16B, v14.16B, #8 // .....................................* // --------- cycle (expected) ----------> // 0 25 // |------------------------|------------ // rev64 V.16B, v29.16B // ...*.................................. // aesr v31.16b, v26.16b // .......*.............................. // pmull2 V.1q, v3.2d, v10.2d // .*.................................... // eor V.16B, v9.16B, v0.16B // .*.................................... // str w26, [sp, #STACK_BASE_AES_ST + 60] // *..................................... // ext V.16B, V.16B, V.16B, #8 // .........*............................ // aesr v31.16b, v27.16b // ..........................*........... // pmull V.1q, V.1d, v12.1d // .........*............................ // ldr Q, [sp, #STACK_BASE_AES_ST + 48] // ..*................................... // aesr v31.16b, v28.16b // ............................*......... // eor V.16B, v4.16B, V.16B // ............*......................... // pmull2 V.1q, V.2d, v12.2d // .....*................................ // eor V.16B, V.16B, V.16B // ...........*.......................... // aese v31.16b, v15.16b // ..............................*....... // eor V.16B, V.16B, v17.16B // ...*.................................. // aesr V.16b, v18.16b // ......*............................... // eor V.16B, v31.16B, v16.16B // ................................*..... // ext V.16B, V.16B, V.16B, #8 // ......*............................... // aesr V.16b, v19.16b // ........*............................. // eor V.16B, V.16B, v5.16B // ..................................*... // eor V.16B, v11.16B, V.16B // ........*............................. // aesr V.16b, v20.16b // ..........*........................... // mov D, v3.d[1] // *..................................... // aesr V.16b, v21.16b // ............*......................... // str q1, [x2, #32] // ..............................*....... // ext V.16B, V.16B, V.16B, #8 // ..........*........................... // aesr V.16b, v22.16b // ..............*....................... // ldr Q, [x6, #16] // *..................................... // eor V.8B, V.8B, v3.8B // ..*................................... // aesr V.16b, v23.16b // ................*..................... // eor V.16B, V.16B, V.16B // ...............*...................... // str Q, [x2], #(4*16) // ....................................*. // aesr V.16b, v24.16b // ..................*................... // aesr V.16b, v25.16b // ....................*................. // str q13, [x2, #-80] // ............................*......... // pmull2 V.1q, V.2d, V.2d // .............*........................ // aesr V.16b, v26.16b // .......................*.............. // aesr V.16b, v27.16b // .........................*............ // mov D, v14.d[1] // ..............*....................... // aesr V.16b, v28.16b // ...........................*.......... // eor V.16B, V.16B, V.16B // .......*.............................. // ldr Q, [x6, #48] // .....*................................ // aese V.16b, v15.16b // .............................*........ // ldr Q, [x6, #80] // ....*................................. // pmull V.1q, V.1d, V.1d // ....*................................. // ldr Q, [x6, #64] // ................*..................... // eor V.16B, V.16B, v16.16B // ...............................*...... // eor V.16B, V.16B, V.16B // ...................*.................. // pmull V.1q, v14.1d, V.1d // .................*.................... // eor V.16B, V.16B, v2.16B // .................................*.... // eor V.16B, V.16B, V.16B // ....................*................. // pmull2 V.1q, v14.2d, V.2d // ...........*.......................... // eor V.8B, V.8B, v14.8B // ..................*................... // pmull V.1q, V.1d, V.1d // .....................*................ // eor V.16B, V.16B, V.16B // .......................*.............. // eor V.16B, V.16B, V.16B // .............*........................ // pmull V.1q, V.1d, V.1d // ...................*.................. // pmull2 V.1q, V.2d, V.2d // ...............*...................... // pmull2 V.1q, V.2d, V.2d // ......................*............... // eor V.16B, V.16B, V.16B // .................*.................... // eor V.16B, V.16B, V.16B // .........................*............ // eor V.16B, V.16B, V.16B // ......................*............... // pmull V.1q, V.1d, v7.1d // ........................*............. // ext V.16B, V.16B, V.16B, #8 // .....................*................ // eor V.16B, V.16B, V.16B // ........................*............. // eor V.16B, V.16B, V.16B // ..........................*........... // eor V.16B, V.16B, V.16B // ...........................*.......... // eor V.16B, V.16B, V.16B // .............................*........ // ext V.16B, V.16B, V.16B, #8 // ................................*..... // pmull V.1q, V.1d, v7.1d // ...............................*...... // eor V.16B, V.16B, V.16B // .................................*.... // eor V.16B, V.16B, V.16B // ...................................*.. // ext v30.16B, V.16B, V.16B, #8 // .....................................* // str Q, [x2, #-16] // ...................................*.. b Lloop_unrolled_start_end Lloop_unrolled_start_iter_1: ldr q12, [x6] ldr q13, [x6, #32] ldr q14, [x6, #16] ldr q29, [x0, #(3*16)] add w14, w13, #3 rev w14, w14 str w14, [sp, #(STACK_BASE_AES_ST + 3*16 + 12)] // @slothy:writes=stack_3 ldr q0, [sp, #(STACK_BASE_AES_ST + 3*16)] // @slothy:reads=stack_3 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aesr v0.16b, v27.16b aesr v0.16b, v28.16b aese v0.16b, v15.16b eor v0.16b, v0.16b, v16.16b eor v0.16b, v0.16b, v29.16b str q0, [x2, #(3*16)] rev64 v29.16b, v29.16b // Low product pmull v8.1q, v29.1d, v12.1d // High product pmull2 v9.1q, v29.2d, v12.2d // Middle product mov d11, v29.d[1] eor v11.8b, v11.8b, v29.8b pmull v10.1q, v11.1d, v14.1d ldr q29, [x0, #(2*16)] add w14, w13, #2 rev w14, w14 str w14, [sp, #(STACK_BASE_AES_ST + 2*16 + 12)] // @slothy:writes=stack_2 ldr q0, [sp, #(STACK_BASE_AES_ST + 2*16)] // @slothy:reads=stack_2 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aesr v0.16b, v27.16b aesr v0.16b, v28.16b aese v0.16b, v15.16b eor v0.16b, v0.16b, v16.16b eor v0.16b, v0.16b, v29.16b str q0, [x2, #(2*16)] rev64 v29.16b, v29.16b // Low product pmull v11.1q, v29.1d, v13.1d eor v8.16b, v8.16b, v11.16b // High product pmull2 v11.1q, v29.2d, v13.2d eor v9.16b, v9.16b, v11.16b // Middle product ext v11.16b, v29.16b, v29.16b, #8 eor v11.16b, v11.16b, v29.16b pmull2 v11.1q, v11.2d, v14.2d eor v10.16b, v10.16b, v11.16b ldr q12, [x6, #48] ldr q13, [x6, #80] ldr q14, [x6, #64] ldr q29, [x0, #(1*16)] add w14, w13, #1 rev w14, w14 str w14, [sp, #(STACK_BASE_AES_ST + 1*16 + 12)] // @slothy:writes=stack_1 ldr q0, [sp, #(STACK_BASE_AES_ST + 1*16)] // @slothy:reads=stack_1 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aesr v0.16b, v27.16b aesr v0.16b, v28.16b aese v0.16b, v15.16b eor v0.16b, v0.16b, v16.16b eor v0.16b, v0.16b, v29.16b str q0, [x2, #(1*16)] rev64 v29.16b, v29.16b // Low product pmull v11.1q, v29.1d, v12.1d eor v8.16b, v8.16b, v11.16b // High product pmull2 v11.1q, v29.2d, v12.2d eor v9.16b, v9.16b, v11.16b // Middle product mov d11, v29.d[1] eor v11.8b, v11.8b, v29.8b pmull v11.1q, v11.1d, v14.1d eor v10.16b, v10.16b, v11.16b ldr q29, [x0], #(4*16) add w14, w13, #0 rev w14, w14 str w14, [sp, #(STACK_BASE_AES_ST + 0*16 + 12)] // @slothy:writes=stack_0 ldr q0, [sp, #(STACK_BASE_AES_ST + 0*16)] // @slothy:reads=stack_0 aesr v0.16b, v18.16b aesr v0.16b, v19.16b aesr v0.16b, v20.16b aesr v0.16b, v21.16b aesr v0.16b, v22.16b aesr v0.16b, v23.16b aesr v0.16b, v24.16b aesr v0.16b, v25.16b aesr v0.16b, v26.16b aesr v0.16b, v27.16b aesr v0.16b, v28.16b aese v0.16b, v15.16b eor v0.16b, v0.16b, v16.16b eor v0.16b, v0.16b, v29.16b str q0, [x2], #(4*16) rev64 v29.16b, v29.16b eor v29.16b, v29.16b, v30.16b // Low product pmull v11.1q, v29.1d, v13.1d eor v8.16b, v8.16b, v11.16b // High product pmull2 v11.1q, v29.2d, v13.2d eor v9.16b, v9.16b, v11.16b // Middle product ext v11.16b, v29.16b, v29.16b, #8 eor v11.16b, v11.16b, v29.16b pmull2 v11.1q, v11.2d, v14.2d eor v10.16b, v10.16b, v11.16b eor v0.16b, v8.16b, v9.16b pmull v1.1q, v9.1d, v7.1d ext v9.16b, v9.16b, v9.16b, #8 eor v10.16b, v10.16b, v0.16b eor v1.16b, v9.16b, v1.16b eor v10.16b, v10.16b, v1.16b pmull v9.1q, v10.1d, v7.1d eor v8.16b, v8.16b, v9.16b ext v10.16b, v10.16b, v10.16b, #8 eor v30.16b, v8.16b, v10.16b ext v30.16b, v30.16b, v30.16b, #8 add w13, w13, #UNROLL Lloop_unrolled_start_iter_1_end: Lloop_unrolled_start_end: Lloop_unrolled_end: load_htable_12 cbz remainder, Lloop_1x_end Lloop_1x_start: ldr plain_q, [input], #16 aes_full_block aes_st, plain, res, 0 str res_q, [output], #16 ghash_init_with_tag_0 plain, Ht1, Ht12, tag ghash_finalize tag add ctr, ctr, #1 sub remainder, remainder, #1 cbnz remainder, Lloop_1x_start Lloop_1x_end: // Return number of bytes processed mov x0, byte_len // Store new authentication tag rev64 tag.16b, tag.16b str tag_q, [tag_ptr] // Store updated counter // rev32 rtmp_ctr.16b, rtmp_ctr.16b // str rtmp_ctr_q, [ivec] rev ctr_tmp_w, ctr str ctr_tmp_w, [ivec, #12] restore_vregs restore_gprs Ldec_postamble_end: add sp, sp, #STACK_SIZE ret #endif #endif // !OPENSSL_NO_ASM && defined(__AARCH64EL__) && defined(__APPLE__) #if defined(__ELF__) // See https: // www.airs.com/blog/archives/518. .section .note.GNU-stack,"",%progbits #endif